diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 6f9c5cfe1107bc..466a74d50d8697 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2416,21 +2416,21 @@ const options: RenovateOptions[] = [ }, { name: 'concurrentRequestLimit', - description: 'Limit concurrent requests per host. Set to 0 for no limit.', + description: 'Limit concurrent requests per host.', type: 'integer', - default: 5, stage: 'repository', parents: ['hostRules'], + default: null, cli: false, env: false, }, { name: 'maxRequestsPerSecond', - description: 'Limit requests rate per host. Set to 0 for no limit.', + description: 'Limit requests rate per host.', type: 'integer', - default: 5, stage: 'repository', parents: ['hostRules'], + default: 0, cli: false, env: false, }, diff --git a/lib/util/http/host-rules.spec.ts b/lib/util/http/host-rules.spec.ts index d558558629995a..2b452132838578 100644 --- a/lib/util/http/host-rules.spec.ts +++ b/lib/util/http/host-rules.spec.ts @@ -3,12 +3,7 @@ import { bootstrap } from '../../proxy'; import type { HostRule } from '../../types'; import * as hostRules from '../host-rules'; import { dnsLookup } from './dns'; -import { - applyHostRule, - findMatchingRule, - getConcurrentRequestsLimit, - getThrottleIntervalMs, -} from './host-rules'; +import { applyHostRule, findMatchingRule } from './host-rules'; import type { GotOptions } from './types'; const url = 'https://github.com'; @@ -565,48 +560,4 @@ describe('util/http/host-rules', () => { }, }); }); - - describe('getConcurrentRequestsLimit', () => { - it('returns default value for undefined rule', () => { - expect(getConcurrentRequestsLimit('https://example.com', 42)).toBe(42); - }); - - it('returns null for 0', () => { - hostRules.add({ - matchHost: 'https://example.com', - concurrentRequestLimit: 0, - }); - expect(getConcurrentRequestsLimit('https://example.com', 42)).toBeNull(); - }); - - it('returns positive limit', () => { - hostRules.add({ - matchHost: 'https://example.com', - concurrentRequestLimit: 143, - }); - expect(getConcurrentRequestsLimit('https://example.com', 42)).toBe(143); - }); - }); - - describe('getThrottleIntervalMs', () => { - it('returns default value for undefined rule', () => { - expect(getThrottleIntervalMs('https://example.com', 42)).toBe(24); // 1000 / 42 - }); - - it('returns null for 0', () => { - hostRules.add({ - matchHost: 'https://example.com', - maxRequestsPerSecond: 0, - }); - expect(getThrottleIntervalMs('https://example.com', 42)).toBeNull(); - }); - - it('returns positive limit', () => { - hostRules.add({ - matchHost: 'https://example.com', - maxRequestsPerSecond: 143, - }); - expect(getThrottleIntervalMs('https://example.com', 42)).toBe(7); // 1000 / 143 - }); - }); }); diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index 54387c4ec3bb86..20c898103743a2 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -217,36 +217,16 @@ export function applyHostRule( return options; } -export function getConcurrentRequestsLimit( - url: string, - defaultValue: number | null, -): number | null { +export function getConcurrentRequestsLimit(url: string): number | null { const { concurrentRequestLimit } = hostRules.find({ url }); - - if (!is.number(concurrentRequestLimit)) { - return defaultValue; - } - - if (concurrentRequestLimit > 0) { - return concurrentRequestLimit; - } - - return null; + return is.number(concurrentRequestLimit) && concurrentRequestLimit > 0 + ? concurrentRequestLimit + : null; } -export function getThrottleIntervalMs( - url: string, - defaultValue: number | null, -): number | null { +export function getThrottleIntervalMs(url: string): number | null { const { maxRequestsPerSecond } = hostRules.find({ url }); - - if (!is.number(maxRequestsPerSecond)) { - return defaultValue ? Math.ceil(1000 / defaultValue) : defaultValue; // 5 requests per second - } - - if (maxRequestsPerSecond > 0) { - return Math.ceil(1000 / maxRequestsPerSecond); - } - - return null; + return is.number(maxRequestsPerSecond) && maxRequestsPerSecond > 0 + ? Math.ceil(1000 / maxRequestsPerSecond) + : null; } diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts index 453cbc032cdbcc..4541f6b86a2af9 100644 --- a/lib/util/http/index.spec.ts +++ b/lib/util/http/index.spec.ts @@ -516,68 +516,34 @@ describe('util/http/index', () => { }); describe('Throttling', () => { - const delta = 100; - - beforeEach(() => { - jest.useFakeTimers({ advanceTimers: true }); - }); - afterEach(() => { jest.useRealTimers(); }); it('works without throttling', async () => { - httpMock.scope(baseUrl).get('/foo').times(10).reply(200, 'bar'); + jest.useFakeTimers({ advanceTimers: 1 }); + httpMock.scope(baseUrl).get('/foo').twice().reply(200, 'bar'); const t1 = Date.now(); - await Promise.all([ - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - ]); + await http.get('http://renovate.com/foo'); + await http.get('http://renovate.com/foo'); const t2 = Date.now(); - expect(t2 - t1).toBeLessThan(delta); + expect(t2 - t1).toBeLessThan(100); }); it('limits request rate by host', async () => { - httpMock.scope(baseUrl).get('/foo').times(4).reply(200, 'bar'); - hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: 3 }); - - const t1 = Date.now(); - await Promise.all([ - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - ]); - const t2 = Date.now(); - - expect(t2 - t1).toBeWithin(1000, 1000 + delta); - }); - - it('defaults to 5 requests per second', async () => { - Http.setDefaultLimits(); - httpMock.scope(baseUrl).get('/foo').times(5).reply(200, 'bar'); + jest.useFakeTimers({ advanceTimers: true }); + httpMock.scope(baseUrl).get('/foo').twice().reply(200, 'bar'); + hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: 0.25 }); const t1 = Date.now(); - await Promise.all([ - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - http.get('http://renovate.com/foo'), - ]); + await http.get('http://renovate.com/foo'); + jest.advanceTimersByTime(4000); + await http.get('http://renovate.com/foo'); const t2 = Date.now(); - expect(t2 - t1).toBeWithin(800, 800 + delta); + expect(t2 - t1).toBeGreaterThanOrEqual(4000); }); }); diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 920620fb1086aa..c58279d4210ccc 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -126,14 +126,6 @@ async function gotTask( } export class Http { - private static defaultConcurrentRequestLimit: number | null = null; - private static defaultMaxRequestsPerSecond: number | null = null; - - static setDefaultLimits(): void { - Http.defaultConcurrentRequestLimit = 5; - Http.defaultMaxRequestsPerSecond = 5; - } - private options?: GotOptions; constructor( @@ -151,7 +143,7 @@ export class Http { } protected getThrottle(url: string): Throttle | null { - return getThrottle(url, Http.defaultMaxRequestsPerSecond); + return getThrottle(url); } protected async request( @@ -257,7 +249,7 @@ export class Http { ? () => throttle.add>(httpTask) : httpTask; - const queue = getQueue(url, Http.defaultConcurrentRequestLimit); + const queue = getQueue(url); const queuedTask: GotTask = queue ? () => queue.add>(throttledTask) : throttledTask; diff --git a/lib/util/http/queue.spec.ts b/lib/util/http/queue.spec.ts index ac8f12c0d5e9a7..69069522393daa 100644 --- a/lib/util/http/queue.spec.ts +++ b/lib/util/http/queue.spec.ts @@ -12,15 +12,15 @@ describe('util/http/queue', () => { }); it('returns null for invalid URL', () => { - expect(getQueue('$#@!', null)).toBeNull(); + expect(getQueue('$#@!')).toBeNull(); }); it('returns queue for valid url', () => { - const q1a = getQueue('https://example.com', null); - const q1b = getQueue('https://example.com', null); + const q1a = getQueue('https://example.com'); + const q1b = getQueue('https://example.com'); - const q2a = getQueue('https://example.com:8080', null); - const q2b = getQueue('https://example.com:8080', null); + const q2a = getQueue('https://example.com:8080'); + const q2b = getQueue('https://example.com:8080'); expect(q1a).not.toBeNull(); expect(q1a).toBe(q1b); diff --git a/lib/util/http/queue.ts b/lib/util/http/queue.ts index 44beff21ef8c57..1c127d2f265606 100644 --- a/lib/util/http/queue.ts +++ b/lib/util/http/queue.ts @@ -5,10 +5,7 @@ import { getConcurrentRequestsLimit } from './host-rules'; const hostQueues = new Map(); -export function getQueue( - url: string, - defaultConcurrentRequestLimit: number | null, -): PQueue | null { +export function getQueue(url: string): PQueue | null { const host = parseUrl(url)?.host; if (!host) { // should never happen @@ -19,10 +16,7 @@ export function getQueue( let queue = hostQueues.get(host); if (queue === undefined) { queue = null; // null represents "no queue", as opposed to undefined - const concurrency = getConcurrentRequestsLimit( - url, - defaultConcurrentRequestLimit, - ); + const concurrency = getConcurrentRequestsLimit(url); if (concurrency) { logger.debug(`Using queue: host=${host}, concurrency=${concurrency}`); queue = new PQueue({ concurrency }); diff --git a/lib/util/http/throttle.spec.ts b/lib/util/http/throttle.spec.ts index d0e3a2f1ce4cb8..5163f23defb177 100644 --- a/lib/util/http/throttle.spec.ts +++ b/lib/util/http/throttle.spec.ts @@ -12,15 +12,15 @@ describe('util/http/throttle', () => { }); it('returns null for invalid URL', () => { - expect(getThrottle('$#@!', null)).toBeNull(); + expect(getThrottle('$#@!')).toBeNull(); }); it('returns throttle for valid url', () => { - const t1a = getThrottle('https://example.com', null); - const t1b = getThrottle('https://example.com', null); + const t1a = getThrottle('https://example.com'); + const t1b = getThrottle('https://example.com'); - const t2a = getThrottle('https://example.com:8080', null); - const t2b = getThrottle('https://example.com:8080', null); + const t2a = getThrottle('https://example.com:8080'); + const t2b = getThrottle('https://example.com:8080'); expect(t1a).not.toBeNull(); expect(t1a).toBe(t1b); diff --git a/lib/util/http/throttle.ts b/lib/util/http/throttle.ts index 995a6c0765ab9e..c868354ce58837 100644 --- a/lib/util/http/throttle.ts +++ b/lib/util/http/throttle.ts @@ -22,10 +22,7 @@ export class Throttle { } } -export function getThrottle( - url: string, - defaultMaxRequestsPerSecond: number | null, -): Throttle | null { +export function getThrottle(url: string): Throttle | null { const host = parseUrl(url)?.host; if (!host) { // should never happen @@ -36,10 +33,7 @@ export function getThrottle( let throttle = hostThrottles.get(host); if (throttle === undefined) { throttle = null; // null represents "no throttle", as opposed to undefined - const throttleOptions = getThrottleIntervalMs( - url, - defaultMaxRequestsPerSecond, - ); + const throttleOptions = getThrottleIntervalMs(url); if (throttleOptions) { const intervalMs = throttleOptions; logger.debug(`Using throttle ${intervalMs} intervalMs for host ${host}`); diff --git a/lib/workers/global/initialize.ts b/lib/workers/global/initialize.ts index d2b5d0272f2415..47c780b4ea4049 100644 --- a/lib/workers/global/initialize.ts +++ b/lib/workers/global/initialize.ts @@ -10,7 +10,6 @@ import * as packageCache from '../../util/cache/package'; import { setEmojiConfig } from '../../util/emoji'; import { validateGitVersion } from '../../util/git'; import * as hostRules from '../../util/host-rules'; -import { Http } from '../../util/http'; import { initMergeConfidence } from '../../util/merge-confidence'; import { setMaxLimit } from './limits'; @@ -80,7 +79,6 @@ export async function globalInitialize( config_: AllConfig, ): Promise { let config = config_; - Http.setDefaultLimits(); await checkVersions(); setGlobalHostRules(config); config = await initPlatform(config);