From 5719af0a2cde891f475277f34c7b5bf17b881265 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 12 Jun 2020 13:07:58 +0200 Subject: [PATCH 1/7] refactor: move caching to util/http --- lib/util/clone.ts | 2 +- lib/util/got/cache-get.ts | 53 --------------------------------------- lib/util/got/index.ts | 3 +-- lib/util/http/index.ts | 37 +++++++++++++++++++++++++-- 4 files changed, 37 insertions(+), 58 deletions(-) delete mode 100644 lib/util/got/cache-get.ts diff --git a/lib/util/clone.ts b/lib/util/clone.ts index 435e4b050914de..d1c9530e7e597a 100644 --- a/lib/util/clone.ts +++ b/lib/util/clone.ts @@ -1,5 +1,5 @@ import safeStringify from 'fast-safe-stringify'; -export function clone(input: T): T { +export function clone(input: T = null): T { return JSON.parse(safeStringify(input)); } diff --git a/lib/util/got/cache-get.ts b/lib/util/got/cache-get.ts deleted file mode 100644 index c49ee5d9bf5f5e..00000000000000 --- a/lib/util/got/cache-get.ts +++ /dev/null @@ -1,53 +0,0 @@ -import crypto from 'crypto'; -import { logger } from '../../logger'; -import * as runCache from '../cache/run'; -import { clone } from '../clone'; -import { create } from './util'; - -// With this caching, it means every GET request is cached during each repository run - -function cloneBody(response: any): any { - return { - ...response, - body: clone(response.body), - }; -} - -export default create({ - options: {}, - handler: (options, next) => { - if (options.stream) { - return next(options); - } - if (!['github', 'npm'].includes(options.hostType)) { - return next(options).then(cloneBody); - } - if (options.method === 'GET') { - const cacheKey = crypto - .createHash('md5') - .update( - 'got-' + - JSON.stringify({ href: options.href, headers: options.headers }) - ) - .digest('hex'); - if (options.useCache === false) { - logger.trace('GET cache skipped: ' + options.href); - } else { - const cachedGot = runCache.get(cacheKey); - // istanbul ignore if - if (cachedGot) { - logger.trace('GET cache hit: ' + options.href); - return cachedGot; - } - logger.trace('GET cache miss: ' + options.href); - } - const promisedRes = next(options).catch((err) => { - runCache.set(cacheKey, null); - throw err; - }); - runCache.set(cacheKey, promisedRes); - return promisedRes.then(cloneBody); - } - return next(options); - }, -}); diff --git a/lib/util/got/index.ts b/lib/util/got/index.ts index 66ee31e67442a9..c385a9efeeafd0 100644 --- a/lib/util/got/index.ts +++ b/lib/util/got/index.ts @@ -1,6 +1,5 @@ import got from 'got'; import auth from './auth'; -import cacheGet from './cache-get'; import hostRules from './host-rules'; import { mergeInstances } from './util'; @@ -11,6 +10,6 @@ export * from './common'; * - Cache all GET requests for the lifetime of the repo * */ -export const api = mergeInstances(got, cacheGet, hostRules, auth); +export const api = mergeInstances(got, hostRules, auth); export default api; diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 5521261b9dfa68..eeb5488ddd0c73 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -1,4 +1,8 @@ +import crypto from 'crypto'; import URL from 'url'; +import { GotPromise } from 'got'; +import * as runCache from '../cache/run'; +import { clone } from '../clone'; import got from '../got'; interface OutgoingHttpHeaders { @@ -28,6 +32,14 @@ export interface HttpResponse { headers: any; } +function cloneResponse(response: any): any { + // clone body and headers so that the cached result doesn't get accidentally mutated + return { + body: clone(response.body), + headers: clone(response.headers), + }; +} + export class Http { constructor(private hostType: string, private options?: HttpOptions) {} @@ -73,8 +85,29 @@ export class Http { process.env.RENOVATE_USER_AGENT || 'https://github.com/renovatebot/renovate', }; - const res = await got(resolvedUrl, combinedOptions); - return { body: res.body, headers: res.headers }; + + // Cache GET requests unless useCache=false + let promisedRes: GotPromise; + if (combinedOptions.method === 'get') { + const cacheKey = crypto + .createHash('md5') + .update( + 'got-' + + JSON.stringify({ url: resolvedUrl, headers: options.headers }) + ) + .digest('hex'); + if (combinedOptions.useCache !== false) { + // check cache unless bypassing it + promisedRes = runCache.get(cacheKey); + } + if (promisedRes === undefined) { + // cache miss OR cache bypass + promisedRes = got(resolvedUrl, combinedOptions); + } + runCache.set(cacheKey, promisedRes); // always set + return cloneResponse(await promisedRes); + } + return cloneResponse(await got(resolvedUrl, combinedOptions)); } get(url: string, options: HttpOptions = {}): Promise { From ef869240ae60e09acfeb272909c930cd1949d212 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 12 Jun 2020 13:52:53 +0200 Subject: [PATCH 2/7] refactor await --- lib/util/http/index.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index c7df26f6f9efc9..436767fc541bc5 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -32,7 +32,8 @@ export interface HttpResponse { headers: any; } -function cloneResponse(response: any): any { +async function cloneResponse(promisedResponse: Promise): Promise { + const response = await promisedResponse; // clone body and headers so that the cached result doesn't get accidentally mutated return { body: clone(response.body), @@ -43,7 +44,7 @@ function cloneResponse(response: any): any { export class Http { constructor(private hostType: string, private options?: HttpOptions) {} - protected async request( + protected request( requestUrl: string | URL, httpOptions?: InternalHttpOptions ): Promise | null> { @@ -101,9 +102,9 @@ export class Http { promisedRes = got(url, options); } runCache.set(cacheKey, promisedRes); // always set - return cloneResponse(await promisedRes); + return cloneResponse(promisedRes); } - return cloneResponse(await got(url, options)); + return cloneResponse(got(url, options)); } get(url: string, options: HttpOptions = {}): Promise { From 62f48ee88d204d3651226892075b61df2bf213d7 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 12 Jun 2020 13:56:21 +0200 Subject: [PATCH 3/7] types --- lib/util/http/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 436767fc541bc5..290cf700962b57 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -32,7 +32,9 @@ export interface HttpResponse { headers: any; } -async function cloneResponse(promisedResponse: Promise): Promise { +async function cloneResponse( + promisedResponse: GotPromise +): Promise> { const response = await promisedResponse; // clone body and headers so that the cached result doesn't get accidentally mutated return { From 904e7699712965ae3666fcf8acceffa012385b2f Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 12 Jun 2020 14:00:39 +0200 Subject: [PATCH 4/7] improve types, add todo --- lib/util/http/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 290cf700962b57..f81af8d9af0c4e 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -32,13 +32,13 @@ export interface HttpResponse { headers: any; } -async function cloneResponse( +async function cloneResponse( promisedResponse: GotPromise -): Promise> { +): Promise> { const response = await promisedResponse; // clone body and headers so that the cached result doesn't get accidentally mutated return { - body: clone(response.body), + body: clone(response.body), // TODO: is this ok if the body is *not* JSON ? headers: clone(response.headers), }; } @@ -104,9 +104,9 @@ export class Http { promisedRes = got(url, options); } runCache.set(cacheKey, promisedRes); // always set - return cloneResponse(promisedRes); + return cloneResponse(promisedRes); } - return cloneResponse(got(url, options)); + return cloneResponse(got(url, options)); } get(url: string, options: HttpOptions = {}): Promise { From 58b487aca392625f11bb8f501250cf3a2bebaa96 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 12 Jun 2020 14:47:25 +0200 Subject: [PATCH 5/7] update tests --- lib/util/got/__snapshots__/index.spec.ts.snap | 42 +++++++++---------- lib/util/got/index.spec.ts | 27 +----------- 2 files changed, 22 insertions(+), 47 deletions(-) diff --git a/lib/util/got/__snapshots__/index.spec.ts.snap b/lib/util/got/__snapshots__/index.spec.ts.snap index d3e89bf57f2d4b..fc49174cb79403 100644 --- a/lib/util/got/__snapshots__/index.spec.ts.snap +++ b/lib/util/got/__snapshots__/index.spec.ts.snap @@ -1,18 +1,14 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`util/got/index uses basic auth 1`] = ` +exports[`util/got/index gets 1`] = ` Object { "body": Object {}, "options": Object { - "auth": ":test", "baseUrl": "https://api.github.com/", "cache": false, "decompress": true, "followRedirect": true, "form": false, - "gotTimeout": Object { - "request": 60000, - }, "hash": "", "headers": Object { "accept": "application/json", @@ -27,6 +23,7 @@ Object { "beforeRetry": Array [], "init": Array [], }, + "hostType": "github", "hostname": "api.github.com", "href": "https://api.github.com/some", "json": true, @@ -43,24 +40,21 @@ Object { "search": "", "stream": false, "throwHttpErrors": true, + "useCache": false, "useElectronNet": false, }, } `; -exports[`util/got/index uses basic auth 2`] = ` +exports[`util/got/index gets 2`] = ` Object { "body": Object {}, "options": Object { - "auth": ":test", "baseUrl": "https://api.github.com/", "cache": false, "decompress": true, "followRedirect": true, "form": false, - "gotTimeout": Object { - "request": 60000, - }, "hash": "", "headers": Object { "accept": "application/json", @@ -75,10 +69,11 @@ Object { "beforeRetry": Array [], "init": Array [], }, + "hostType": "github", "hostname": "api.github.com", "href": "https://api.github.com/some", "json": true, - "method": "GET", + "method": "HEAD", "path": "/some", "pathname": "/some", "protocol": "https:", @@ -96,20 +91,23 @@ Object { } `; -exports[`util/got/index uses bearer auth 1`] = ` +exports[`util/got/index uses basic auth 1`] = ` Object { "body": Object {}, "options": Object { + "auth": ":test", "baseUrl": "https://api.github.com/", "cache": false, "decompress": true, "followRedirect": true, "form": false, + "gotTimeout": Object { + "request": 60000, + }, "hash": "", "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", - "authorization": "Bearer XXX", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", }, "hooks": Object { @@ -141,20 +139,23 @@ Object { } `; -exports[`util/got/index uses bearer auth 2`] = ` +exports[`util/got/index uses basic auth 2`] = ` Object { "body": Object {}, "options": Object { + "auth": ":test", "baseUrl": "https://api.github.com/", "cache": false, "decompress": true, "followRedirect": true, "form": false, + "gotTimeout": Object { + "request": 60000, + }, "hash": "", "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", - "authorization": "Bearer XXX", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", }, "hooks": Object { @@ -186,7 +187,7 @@ Object { } `; -exports[`util/got/index uses no cache 1`] = ` +exports[`util/got/index uses bearer auth 1`] = ` Object { "body": Object {}, "options": Object { @@ -199,6 +200,7 @@ Object { "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", + "authorization": "Bearer XXX", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", }, "hooks": Object { @@ -209,7 +211,6 @@ Object { "beforeRetry": Array [], "init": Array [], }, - "hostType": "github", "hostname": "api.github.com", "href": "https://api.github.com/some", "json": true, @@ -226,13 +227,12 @@ Object { "search": "", "stream": false, "throwHttpErrors": true, - "useCache": false, "useElectronNet": false, }, } `; -exports[`util/got/index uses no cache 2`] = ` +exports[`util/got/index uses bearer auth 2`] = ` Object { "body": Object {}, "options": Object { @@ -245,6 +245,7 @@ Object { "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", + "authorization": "Bearer XXX", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", }, "hooks": Object { @@ -255,11 +256,10 @@ Object { "beforeRetry": Array [], "init": Array [], }, - "hostType": "github", "hostname": "api.github.com", "href": "https://api.github.com/some", "json": true, - "method": "HEAD", + "method": "GET", "path": "/some", "pathname": "/some", "protocol": "https:", diff --git a/lib/util/got/index.spec.ts b/lib/util/got/index.spec.ts index 6b9d09317dc02a..a1352e64b41903 100644 --- a/lib/util/got/index.spec.ts +++ b/lib/util/got/index.spec.ts @@ -69,7 +69,7 @@ describe(getName(__filename), () => { expect(req.isDone()).toBe(true); }); - it('uses no cache', async () => { + it('gets', async () => { const req = mock({}) .head('/some') .reply(200, {}) @@ -93,29 +93,4 @@ describe(getName(__filename), () => { expect(req.isDone()).toBe(true); }); - - it('streams no cache', async () => { - const req = mock(); - - const stream = api.stream('/some', { - baseUrl, - }); - expect(stream).toBeDefined(); - - let data = ''; - - stream.on('data', (c) => { - data += c; - }); - - const done = new Promise((resolve, reject) => { - stream.on('end', resolve); - stream.on('error', reject); - }); - - await done; - - expect(data).toBe('{}'); - expect(req.isDone()).toBe(true); - }); }); From d2f8354af57a718bfde5a6b80d5044b8f4288189 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 12 Jun 2020 15:07:29 +0200 Subject: [PATCH 6/7] drop got auth testing --- lib/util/got/__snapshots__/index.spec.ts.snap | 278 ------------------ lib/util/got/index.spec.ts | 39 +-- 2 files changed, 1 insertion(+), 316 deletions(-) diff --git a/lib/util/got/__snapshots__/index.spec.ts.snap b/lib/util/got/__snapshots__/index.spec.ts.snap index fc49174cb79403..a07baa889b3cc3 100644 --- a/lib/util/got/__snapshots__/index.spec.ts.snap +++ b/lib/util/got/__snapshots__/index.spec.ts.snap @@ -90,281 +90,3 @@ Object { }, } `; - -exports[`util/got/index uses basic auth 1`] = ` -Object { - "body": Object {}, - "options": Object { - "auth": ":test", - "baseUrl": "https://api.github.com/", - "cache": false, - "decompress": true, - "followRedirect": true, - "form": false, - "gotTimeout": Object { - "request": 60000, - }, - "hash": "", - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", - }, - "hooks": Object { - "afterResponse": Array [], - "beforeError": Array [], - "beforeRedirect": Array [], - "beforeRequest": Array [], - "beforeRetry": Array [], - "init": Array [], - }, - "hostname": "api.github.com", - "href": "https://api.github.com/some", - "json": true, - "method": "GET", - "path": "/some", - "pathname": "/some", - "protocol": "https:", - "retry": Object { - "errorCodes": Set {}, - "methods": Set {}, - "retries": [Function], - "statusCodes": Set {}, - }, - "search": "", - "stream": false, - "throwHttpErrors": true, - "useElectronNet": false, - }, -} -`; - -exports[`util/got/index uses basic auth 2`] = ` -Object { - "body": Object {}, - "options": Object { - "auth": ":test", - "baseUrl": "https://api.github.com/", - "cache": false, - "decompress": true, - "followRedirect": true, - "form": false, - "gotTimeout": Object { - "request": 60000, - }, - "hash": "", - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", - }, - "hooks": Object { - "afterResponse": Array [], - "beforeError": Array [], - "beforeRedirect": Array [], - "beforeRequest": Array [], - "beforeRetry": Array [], - "init": Array [], - }, - "hostname": "api.github.com", - "href": "https://api.github.com/some", - "json": true, - "method": "GET", - "path": "/some", - "pathname": "/some", - "protocol": "https:", - "retry": Object { - "errorCodes": Set {}, - "methods": Set {}, - "retries": [Function], - "statusCodes": Set {}, - }, - "search": "", - "stream": false, - "throwHttpErrors": true, - "useElectronNet": false, - }, -} -`; - -exports[`util/got/index uses bearer auth 1`] = ` -Object { - "body": Object {}, - "options": Object { - "baseUrl": "https://api.github.com/", - "cache": false, - "decompress": true, - "followRedirect": true, - "form": false, - "hash": "", - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "authorization": "Bearer XXX", - "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", - }, - "hooks": Object { - "afterResponse": Array [], - "beforeError": Array [], - "beforeRedirect": Array [], - "beforeRequest": Array [], - "beforeRetry": Array [], - "init": Array [], - }, - "hostname": "api.github.com", - "href": "https://api.github.com/some", - "json": true, - "method": "GET", - "path": "/some", - "pathname": "/some", - "protocol": "https:", - "retry": Object { - "errorCodes": Set {}, - "methods": Set {}, - "retries": [Function], - "statusCodes": Set {}, - }, - "search": "", - "stream": false, - "throwHttpErrors": true, - "useElectronNet": false, - }, -} -`; - -exports[`util/got/index uses bearer auth 2`] = ` -Object { - "body": Object {}, - "options": Object { - "baseUrl": "https://api.github.com/", - "cache": false, - "decompress": true, - "followRedirect": true, - "form": false, - "hash": "", - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "authorization": "Bearer XXX", - "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", - }, - "hooks": Object { - "afterResponse": Array [], - "beforeError": Array [], - "beforeRedirect": Array [], - "beforeRequest": Array [], - "beforeRetry": Array [], - "init": Array [], - }, - "hostname": "api.github.com", - "href": "https://api.github.com/some", - "json": true, - "method": "GET", - "path": "/some", - "pathname": "/some", - "protocol": "https:", - "retry": Object { - "errorCodes": Set {}, - "methods": Set {}, - "retries": [Function], - "statusCodes": Set {}, - }, - "search": "", - "stream": false, - "throwHttpErrors": true, - "useElectronNet": false, - }, -} -`; - -exports[`util/got/index uses private-token auth 1`] = ` -Object { - "body": Object {}, - "options": Object { - "baseUrl": "https://api.github.com/", - "cache": false, - "decompress": true, - "followRedirect": true, - "form": false, - "hash": "", - "headers": Object { - "Private-token": "XXX", - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", - }, - "hooks": Object { - "afterResponse": Array [], - "beforeError": Array [], - "beforeRedirect": Array [], - "beforeRequest": Array [], - "beforeRetry": Array [], - "init": Array [], - }, - "hostType": "gitlab", - "hostname": "api.github.com", - "href": "https://api.github.com/some", - "json": true, - "method": "GET", - "path": "/some", - "pathname": "/some", - "protocol": "https:", - "retry": Object { - "errorCodes": Set {}, - "methods": Set {}, - "retries": [Function], - "statusCodes": Set {}, - }, - "search": "", - "stream": false, - "throwHttpErrors": true, - "useElectronNet": false, - }, -} -`; - -exports[`util/got/index uses token auth 1`] = ` -Object { - "body": Object {}, - "options": Object { - "baseUrl": "https://api.github.com/", - "cache": false, - "decompress": true, - "followRedirect": true, - "form": false, - "hash": "", - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "authorization": "token XXX", - "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", - }, - "hooks": Object { - "afterResponse": Array [], - "beforeError": Array [], - "beforeRedirect": Array [], - "beforeRequest": Array [], - "beforeRetry": Array [], - "init": Array [], - }, - "hostType": "gitea", - "hostname": "api.github.com", - "href": "https://api.github.com/some", - "json": true, - "method": "GET", - "path": "/some", - "pathname": "/some", - "protocol": "https:", - "retry": Object { - "errorCodes": Set {}, - "methods": Set {}, - "retries": [Function], - "statusCodes": Set {}, - }, - "search": "", - "stream": false, - "throwHttpErrors": true, - "useElectronNet": false, - }, -} -`; diff --git a/lib/util/got/index.spec.ts b/lib/util/got/index.spec.ts index a1352e64b41903..84bd8cf261bd2b 100644 --- a/lib/util/got/index.spec.ts +++ b/lib/util/got/index.spec.ts @@ -1,10 +1,6 @@ import nock from 'nock'; import { getName } from '../../../test/util'; -import { - PLATFORM_TYPE_GITEA, - PLATFORM_TYPE_GITHUB, - PLATFORM_TYPE_GITLAB, -} from '../../constants/platforms'; +import { PLATFORM_TYPE_GITHUB } from '../../constants/platforms'; import * as hostRules from '../host-rules'; import { GotJSONOptions } from './common'; import { api } from '.'; @@ -36,39 +32,6 @@ describe(getName(__filename), () => { return nock(baseUrl, opts).get('/some').times(times).reply(200, {}); } - it('uses bearer auth', async () => { - const req = mock({ reqheaders: { authorization: 'Bearer XXX' } }, 2); - hostRules.add({ baseUrl, token: 'XXX' }); - - expect(await got()).toMatchSnapshot(); - expect(await got({ token: 'XXX' })).toMatchSnapshot(); - expect(req.isDone()).toBe(true); - }); - - it('uses basic auth', async () => { - const req = mock({ reqheaders: { authorization: 'Basic OnRlc3Q=' } }, 2); - - hostRules.add({ password: 'test', timeout: 60000 }); - - expect(await got()).toMatchSnapshot(); - expect(await got({ auth: ':test' })).toMatchSnapshot(); - expect(req.isDone()).toBe(true); - }); - - it('uses token auth', async () => { - const req = mock({ reqheaders: { authorization: 'token XXX' } }); - hostRules.add({ baseUrl, token: 'XXX' }); - expect(await got({ hostType: PLATFORM_TYPE_GITEA })).toMatchSnapshot(); - expect(req.isDone()).toBe(true); - }); - - it('uses private-token auth', async () => { - const req = mock({ reqheaders: { 'private-token': 'XXX' } }); - hostRules.add({ baseUrl, token: 'XXX' }); - expect(await got({ hostType: PLATFORM_TYPE_GITLAB })).toMatchSnapshot(); - expect(req.isDone()).toBe(true); - }); - it('gets', async () => { const req = mock({}) .head('/some') From 796d805310e53cef09f6e3e565f69f1c573cd655 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 12 Jun 2020 17:25:40 +0200 Subject: [PATCH 7/7] feat(internal): apply host-rules in http module Closes #6401 --- .../docker/__snapshots__/index.spec.ts.snap | 12 ++++++ lib/datasource/maven/index.spec.ts | 1 + lib/manager/cargo/artifacts.spec.ts | 1 + lib/manager/gomod/artifacts.spec.ts | 1 + lib/manager/pipenv/artifacts.spec.ts | 1 + lib/util/got/host-rules.ts | 37 ------------------- lib/util/got/index.ts | 3 +- lib/util/http/host-rules.ts | 27 ++++++++++++++ lib/util/http/index.ts | 5 ++- 9 files changed, 48 insertions(+), 40 deletions(-) delete mode 100644 lib/util/got/host-rules.ts create mode 100644 lib/util/http/host-rules.ts diff --git a/lib/datasource/docker/__snapshots__/index.spec.ts.snap b/lib/datasource/docker/__snapshots__/index.spec.ts.snap index df87d8dce223cb..100f3323ec9e7b 100644 --- a/lib/datasource/docker/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/docker/__snapshots__/index.spec.ts.snap @@ -152,6 +152,7 @@ exports[`api/docker getReleases adds library/ prefix for Docker Hub (implicit) 1 Array [ "https://index.docker.io/v2/", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -416,6 +417,7 @@ exports[`api/docker getReleases uses custom registry in depName 1`] = ` Array [ "https://registry.company.com/v2/", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -432,6 +434,7 @@ exports[`api/docker getReleases uses custom registry in depName 1`] = ` Array [ "https://registry.company.com/v2/node/tags/list?n=10000", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -448,6 +451,7 @@ exports[`api/docker getReleases uses custom registry in depName 1`] = ` Array [ "https://registry.company.com/v2/", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -493,6 +497,7 @@ Array [ Array [ "https://registry.company.com/v2/", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -509,6 +514,7 @@ Array [ Array [ "https://registry.company.com/v2/node/tags/list?n=10000", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -525,6 +531,7 @@ Array [ Array [ "https://api.github.com/user/9287/repos?page=3&per_page=100", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -541,6 +548,7 @@ Array [ Array [ "https://registry.company.com/v2/", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -557,6 +565,7 @@ Array [ Array [ "https://registry.company.com/v2/node/manifests/latest", Object { + "auth": "some-username:some-password", "headers": Object { "accept": "application/vnd.docker.distribution.manifest.v2+json", "user-agent": "https://github.com/renovatebot/renovate", @@ -579,6 +588,7 @@ exports[`api/docker getReleases uses lower tag limit for ECR deps 1`] = ` Array [ "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -595,6 +605,7 @@ exports[`api/docker getReleases uses lower tag limit for ECR deps 1`] = ` Array [ "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/node/tags/list?n=1000", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, @@ -611,6 +622,7 @@ exports[`api/docker getReleases uses lower tag limit for ECR deps 1`] = ` Array [ "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", Object { + "auth": "some-username:some-password", "headers": Object { "user-agent": "https://github.com/renovatebot/renovate", }, diff --git a/lib/datasource/maven/index.spec.ts b/lib/datasource/maven/index.spec.ts index f4ac6e99942ff7..0915058b7b5774 100644 --- a/lib/datasource/maven/index.spec.ts +++ b/lib/datasource/maven/index.spec.ts @@ -37,6 +37,7 @@ describe('datasource/maven', () => { hostName: 'frontend_for_private_s3_repository', username: 'username', password: 'password', + timeout: 20000, }); jest.resetAllMocks(); nock.cleanAll(); diff --git a/lib/manager/cargo/artifacts.spec.ts b/lib/manager/cargo/artifacts.spec.ts index 04b3b024baf24c..8d1cbd004dd53f 100644 --- a/lib/manager/cargo/artifacts.spec.ts +++ b/lib/manager/cargo/artifacts.spec.ts @@ -12,6 +12,7 @@ import * as cargo from './artifacts'; jest.mock('fs-extra'); jest.mock('child_process'); jest.mock('../../util/exec/env'); +jest.mock('../../util/http'); const fs: jest.Mocked = _fs as any; const exec: jest.Mock = _exec as any; diff --git a/lib/manager/gomod/artifacts.spec.ts b/lib/manager/gomod/artifacts.spec.ts index 3d782f31046e50..22914528c80993 100644 --- a/lib/manager/gomod/artifacts.spec.ts +++ b/lib/manager/gomod/artifacts.spec.ts @@ -15,6 +15,7 @@ jest.mock('fs-extra'); jest.mock('child_process'); jest.mock('../../util/exec/env'); jest.mock('../../util/host-rules'); +jest.mock('../../util/http'); const fs: jest.Mocked = _fs as any; const exec: jest.Mock = _exec as any; diff --git a/lib/manager/pipenv/artifacts.spec.ts b/lib/manager/pipenv/artifacts.spec.ts index 2be7d6e9848deb..5b176f7f10cfaf 100644 --- a/lib/manager/pipenv/artifacts.spec.ts +++ b/lib/manager/pipenv/artifacts.spec.ts @@ -14,6 +14,7 @@ jest.mock('fs-extra'); jest.mock('child_process'); jest.mock('../../util/exec/env'); jest.mock('../../util/host-rules'); +jest.mock('../../util/http'); const fs: jest.Mocked = _fs as any; const exec: jest.Mock = _exec as any; diff --git a/lib/util/got/host-rules.ts b/lib/util/got/host-rules.ts deleted file mode 100644 index 2c849a3ba95ec1..00000000000000 --- a/lib/util/got/host-rules.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* eslint-disable no-param-reassign */ -import { logger } from '../../logger'; -import * as hostRules from '../host-rules'; -import { create } from './util'; - -// Apply host rules to requests - -export default create({ - options: {}, - handler: (options, next) => { - // istanbul ignore if: never happen? - if (!options.hostname) { - return next(options); - } - const { username, password, token, timeout } = hostRules.find({ - hostType: options.hostType, - url: options.href, - }); - if (options.headers.authorization || options.auth || options.token) { - logger.trace('Authorization already set for host: ' + options.hostname); - } else if (password) { - logger.trace( - 'Applying Basic authentication for host ' + options.hostname - ); - options.auth = `${username || ''}:${password}`; - } else if (token) { - logger.trace( - 'Applying Bearer authentication for host ' + options.hostname - ); - options.token = token; - } - if (timeout) { - options.gotTimeout = { request: timeout }; - } - return next(options); - }, -}); diff --git a/lib/util/got/index.ts b/lib/util/got/index.ts index c385a9efeeafd0..8360e53336db1a 100644 --- a/lib/util/got/index.ts +++ b/lib/util/got/index.ts @@ -1,6 +1,5 @@ import got from 'got'; import auth from './auth'; -import hostRules from './host-rules'; import { mergeInstances } from './util'; export * from './common'; @@ -10,6 +9,6 @@ export * from './common'; * - Cache all GET requests for the lifetime of the repo * */ -export const api = mergeInstances(got, hostRules, auth); +export const api = mergeInstances(got, auth); export default api; diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts new file mode 100644 index 00000000000000..57c6f4e201d920 --- /dev/null +++ b/lib/util/http/host-rules.ts @@ -0,0 +1,27 @@ +import { logger } from '../../logger'; +import * as hostRules from '../host-rules'; + +// Apply host rules to requests + +export function applyHostRules(url: string, inOptions: any): any { + const options = { ...inOptions }; + const foundRules = + hostRules.find({ + hostType: options.hostType, + url, + }) || {}; + const { username, password, token, timeout } = foundRules; + if (options.headers?.authorization || options.auth || options.token) { + logger.trace('Authorization already set for host: ' + options.hostname); + } else if (password) { + logger.trace('Applying Basic authentication for host ' + options.hostname); + options.auth = `${username || ''}:${password}`; + } else if (token) { + logger.trace('Applying Bearer authentication for host ' + options.hostname); + options.token = token; + } + if (timeout) { + options.timeout = timeout; + } + return options; +} diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index ff254197ceda02..2b8cd7d6774d9e 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -4,6 +4,7 @@ import { GotPromise } from 'got'; import * as runCache from '../cache/run'; import { clone } from '../clone'; import got from '../got'; +import { applyHostRules } from './host-rules'; interface OutgoingHttpHeaders { [header: string]: number | string | string[] | undefined; @@ -55,7 +56,7 @@ export class Http { url = URL.resolve(httpOptions.baseUrl, url); } // TODO: deep merge in order to merge headers - const options: any = { + let options: any = { method: 'get', ...this.options, hostType: this.hostType, @@ -88,6 +89,8 @@ export class Http { 'https://github.com/renovatebot/renovate', }; + options = applyHostRules(url, options); + // Cache GET requests unless useCache=false let promisedRes: GotPromise; if (options.method === 'get') {