From 75539c23dcb8be9d85f5be95d49a7e819f28e8e5 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 28 Feb 2024 14:04:27 -0300 Subject: [PATCH 01/14] feat: Throttle and rate limit HTTP by default --- lib/util/http/host-rules.ts | 26 +++++++++++++---- lib/util/http/index.spec.ts | 58 ++++++++++++++++++++++++++++++------- 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index 20c898103743a2..6c173a5edbbab5 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -219,14 +219,28 @@ export function applyHostRule( export function getConcurrentRequestsLimit(url: string): number | null { const { concurrentRequestLimit } = hostRules.find({ url }); - return is.number(concurrentRequestLimit) && concurrentRequestLimit > 0 - ? concurrentRequestLimit - : null; + + if (!is.number(concurrentRequestLimit)) { + return 5; + } + + if (concurrentRequestLimit > 0) { + return concurrentRequestLimit; + } + + return null; } export function getThrottleIntervalMs(url: string): number | null { const { maxRequestsPerSecond } = hostRules.find({ url }); - return is.number(maxRequestsPerSecond) && maxRequestsPerSecond > 0 - ? Math.ceil(1000 / maxRequestsPerSecond) - : null; + + if (!is.number(maxRequestsPerSecond)) { + return Math.ceil(1000 / 5); // 5 requests per second + } + + if (maxRequestsPerSecond > 0) { + return Math.ceil(1000 / maxRequestsPerSecond); + } + + return null; } diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts index 4541f6b86a2af9..f1f7253e34db9e 100644 --- a/lib/util/http/index.spec.ts +++ b/lib/util/http/index.spec.ts @@ -516,34 +516,70 @@ describe('util/http/index', () => { }); describe('Throttling', () => { + const delta = 100; + + beforeEach(() => { + jest.useFakeTimers({ advanceTimers: 1 }); + }); + afterEach(() => { jest.useRealTimers(); }); it('works without throttling', async () => { - jest.useFakeTimers({ advanceTimers: 1 }); - httpMock.scope(baseUrl).get('/foo').twice().reply(200, 'bar'); + httpMock.scope(baseUrl).get('/foo').times(10).reply(200, 'bar'); + hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: -1 }); const t1 = Date.now(); - await http.get('http://renovate.com/foo'); - await http.get('http://renovate.com/foo'); + await Promise.all([ + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + ]); const t2 = Date.now(); - expect(t2 - t1).toBeLessThan(100); + expect(t2 - t1).toBeLessThan(delta); }); it('limits request rate by host', async () => { - jest.useFakeTimers({ advanceTimers: true }); - httpMock.scope(baseUrl).get('/foo').twice().reply(200, 'bar'); + httpMock.scope(baseUrl).get('/foo').times(4).reply(200, 'bar'); hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: 0.25 }); + const advanceTimer = () => jest.advanceTimersByTime(4000); + + const t1 = Date.now(); + await Promise.all([ + http.get('http://renovate.com/foo').then(advanceTimer), + http.get('http://renovate.com/foo').then(advanceTimer), + http.get('http://renovate.com/foo').then(advanceTimer), + http.get('http://renovate.com/foo').then(advanceTimer), + ]); + const t2 = Date.now(); + + expect(t2 - t1).toBeWithin(16000, 16000 + delta); + }); + + it('defaults to 5 requests per second', async () => { + httpMock.scope(baseUrl).get('/foo').times(5).reply(200, 'bar'); + const advanceTimer = () => jest.advanceTimersByTime(200); const t1 = Date.now(); - await http.get('http://renovate.com/foo'); - jest.advanceTimersByTime(4000); - await http.get('http://renovate.com/foo'); + await Promise.all([ + http.get('http://renovate.com/foo').then(advanceTimer), + http.get('http://renovate.com/foo').then(advanceTimer), + http.get('http://renovate.com/foo').then(advanceTimer), + http.get('http://renovate.com/foo').then(advanceTimer), + http.get('http://renovate.com/foo').then(advanceTimer), + ]); const t2 = Date.now(); - expect(t2 - t1).toBeGreaterThanOrEqual(4000); + expect(t2 - t1).toBeWithin(1000, 1000 + delta); }); }); From 732a6d645e79205866489fd8a27559c7a797b151 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 28 Feb 2024 15:43:28 -0300 Subject: [PATCH 02/14] Defaults --- lib/util/http/host-rules.ts | 16 ++++++++++++---- lib/util/http/index.spec.ts | 30 ++++++++++++++---------------- lib/util/http/index.ts | 18 +++++++++++------- lib/util/http/queue.ts | 10 ++++++++-- lib/util/http/throttle.ts | 10 ++++++++-- 5 files changed, 53 insertions(+), 31 deletions(-) diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index 6c173a5edbbab5..6592f9f89db119 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -217,30 +217,38 @@ export function applyHostRule( return options; } -export function getConcurrentRequestsLimit(url: string): number | null { +export function getConcurrentRequestsLimit( + url: string, + defaultValue: number | null, +): number | null { const { concurrentRequestLimit } = hostRules.find({ url }); if (!is.number(concurrentRequestLimit)) { - return 5; + return defaultValue; } if (concurrentRequestLimit > 0) { return concurrentRequestLimit; } + // -1 for reset return null; } -export function getThrottleIntervalMs(url: string): number | null { +export function getThrottleIntervalMs( + url: string, + defaultValue: number | null, +): number | null { const { maxRequestsPerSecond } = hostRules.find({ url }); if (!is.number(maxRequestsPerSecond)) { - return Math.ceil(1000 / 5); // 5 requests per second + return defaultValue ? Math.ceil(1000 / defaultValue) : defaultValue; // 5 requests per second } if (maxRequestsPerSecond > 0) { return Math.ceil(1000 / maxRequestsPerSecond); } + // -1 for reset return null; } diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts index f1f7253e34db9e..453cbc032cdbcc 100644 --- a/lib/util/http/index.spec.ts +++ b/lib/util/http/index.spec.ts @@ -519,7 +519,7 @@ describe('util/http/index', () => { const delta = 100; beforeEach(() => { - jest.useFakeTimers({ advanceTimers: 1 }); + jest.useFakeTimers({ advanceTimers: true }); }); afterEach(() => { @@ -528,7 +528,6 @@ describe('util/http/index', () => { it('works without throttling', async () => { httpMock.scope(baseUrl).get('/foo').times(10).reply(200, 'bar'); - hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: -1 }); const t1 = Date.now(); await Promise.all([ @@ -550,36 +549,35 @@ describe('util/http/index', () => { it('limits request rate by host', async () => { httpMock.scope(baseUrl).get('/foo').times(4).reply(200, 'bar'); - hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: 0.25 }); - const advanceTimer = () => jest.advanceTimersByTime(4000); + hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: 3 }); const t1 = Date.now(); await Promise.all([ - http.get('http://renovate.com/foo').then(advanceTimer), - http.get('http://renovate.com/foo').then(advanceTimer), - http.get('http://renovate.com/foo').then(advanceTimer), - http.get('http://renovate.com/foo').then(advanceTimer), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), ]); const t2 = Date.now(); - expect(t2 - t1).toBeWithin(16000, 16000 + delta); + expect(t2 - t1).toBeWithin(1000, 1000 + delta); }); it('defaults to 5 requests per second', async () => { + Http.setDefaultLimits(); httpMock.scope(baseUrl).get('/foo').times(5).reply(200, 'bar'); - const advanceTimer = () => jest.advanceTimersByTime(200); const t1 = Date.now(); await Promise.all([ - http.get('http://renovate.com/foo').then(advanceTimer), - http.get('http://renovate.com/foo').then(advanceTimer), - http.get('http://renovate.com/foo').then(advanceTimer), - http.get('http://renovate.com/foo').then(advanceTimer), - http.get('http://renovate.com/foo').then(advanceTimer), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), + http.get('http://renovate.com/foo'), ]); const t2 = Date.now(); - expect(t2 - t1).toBeWithin(1000, 1000 + delta); + expect(t2 - t1).toBeWithin(800, 800 + delta); }); }); diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index c58279d4210ccc..4606705cf1a53a 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -17,7 +17,7 @@ import { hooks } from './hooks'; import { applyHostRule, findMatchingRule } from './host-rules'; import { getQueue } from './queue'; import { getRetryAfter, wrapWithRetry } from './retry-after'; -import { Throttle, getThrottle } from './throttle'; +import { getThrottle } from './throttle'; import type { GotJSONOptions, GotOptions, @@ -126,6 +126,14 @@ async function gotTask( } export class Http { + private static defaultConcurrentRequestLimit: number | null = null; + private static defaultMaxRequestsPerSecond: number | null = null; + + static setDefaultLimits(): void { + Http.defaultConcurrentRequestLimit = 5; + Http.defaultMaxRequestsPerSecond = 5; + } + private options?: GotOptions; constructor( @@ -142,10 +150,6 @@ export class Http { }); } - protected getThrottle(url: string): Throttle | null { - return getThrottle(url); - } - protected async request( requestUrl: string | URL, httpOptions: InternalHttpOptions & HttpRequestOptions, @@ -244,12 +248,12 @@ export class Http { }); }; - const throttle = this.getThrottle(url); + const throttle = getThrottle(url, Http.defaultMaxRequestsPerSecond); const throttledTask: GotTask = throttle ? () => throttle.add>(httpTask) : httpTask; - const queue = getQueue(url); + const queue = getQueue(url, Http.defaultConcurrentRequestLimit); const queuedTask: GotTask = queue ? () => queue.add>(throttledTask) : throttledTask; diff --git a/lib/util/http/queue.ts b/lib/util/http/queue.ts index 1c127d2f265606..44beff21ef8c57 100644 --- a/lib/util/http/queue.ts +++ b/lib/util/http/queue.ts @@ -5,7 +5,10 @@ import { getConcurrentRequestsLimit } from './host-rules'; const hostQueues = new Map(); -export function getQueue(url: string): PQueue | null { +export function getQueue( + url: string, + defaultConcurrentRequestLimit: number | null, +): PQueue | null { const host = parseUrl(url)?.host; if (!host) { // should never happen @@ -16,7 +19,10 @@ export function getQueue(url: string): PQueue | null { let queue = hostQueues.get(host); if (queue === undefined) { queue = null; // null represents "no queue", as opposed to undefined - const concurrency = getConcurrentRequestsLimit(url); + const concurrency = getConcurrentRequestsLimit( + url, + defaultConcurrentRequestLimit, + ); if (concurrency) { logger.debug(`Using queue: host=${host}, concurrency=${concurrency}`); queue = new PQueue({ concurrency }); diff --git a/lib/util/http/throttle.ts b/lib/util/http/throttle.ts index c868354ce58837..995a6c0765ab9e 100644 --- a/lib/util/http/throttle.ts +++ b/lib/util/http/throttle.ts @@ -22,7 +22,10 @@ export class Throttle { } } -export function getThrottle(url: string): Throttle | null { +export function getThrottle( + url: string, + defaultMaxRequestsPerSecond: number | null, +): Throttle | null { const host = parseUrl(url)?.host; if (!host) { // should never happen @@ -33,7 +36,10 @@ export function getThrottle(url: string): Throttle | null { let throttle = hostThrottles.get(host); if (throttle === undefined) { throttle = null; // null represents "no throttle", as opposed to undefined - const throttleOptions = getThrottleIntervalMs(url); + const throttleOptions = getThrottleIntervalMs( + url, + defaultMaxRequestsPerSecond, + ); if (throttleOptions) { const intervalMs = throttleOptions; logger.debug(`Using throttle ${intervalMs} intervalMs for host ${host}`); From a096bc1a14e469e3cbd5ddd841e70d9d458af690 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 28 Feb 2024 15:57:15 -0300 Subject: [PATCH 03/14] Initializer --- lib/workers/global/initialize.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/workers/global/initialize.ts b/lib/workers/global/initialize.ts index 47c780b4ea4049..d2b5d0272f2415 100644 --- a/lib/workers/global/initialize.ts +++ b/lib/workers/global/initialize.ts @@ -10,6 +10,7 @@ import * as packageCache from '../../util/cache/package'; import { setEmojiConfig } from '../../util/emoji'; import { validateGitVersion } from '../../util/git'; import * as hostRules from '../../util/host-rules'; +import { Http } from '../../util/http'; import { initMergeConfidence } from '../../util/merge-confidence'; import { setMaxLimit } from './limits'; @@ -79,6 +80,7 @@ export async function globalInitialize( config_: AllConfig, ): Promise { let config = config_; + Http.setDefaultLimits(); await checkVersions(); setGlobalHostRules(config); config = await initPlatform(config); From baba3bb25acdbefe53983eca7b344fe7bae832f8 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 28 Feb 2024 16:22:07 -0300 Subject: [PATCH 04/14] Fix rubygems --- lib/util/http/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 4606705cf1a53a..920620fb1086aa 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -17,7 +17,7 @@ import { hooks } from './hooks'; import { applyHostRule, findMatchingRule } from './host-rules'; import { getQueue } from './queue'; import { getRetryAfter, wrapWithRetry } from './retry-after'; -import { getThrottle } from './throttle'; +import { Throttle, getThrottle } from './throttle'; import type { GotJSONOptions, GotOptions, @@ -150,6 +150,10 @@ export class Http { }); } + protected getThrottle(url: string): Throttle | null { + return getThrottle(url, Http.defaultMaxRequestsPerSecond); + } + protected async request( requestUrl: string | URL, httpOptions: InternalHttpOptions & HttpRequestOptions, @@ -248,7 +252,7 @@ export class Http { }); }; - const throttle = getThrottle(url, Http.defaultMaxRequestsPerSecond); + const throttle = this.getThrottle(url); const throttledTask: GotTask = throttle ? () => throttle.add>(httpTask) : httpTask; From 3d56fa63dbb516f975142aff19a174de888c2b05 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 28 Feb 2024 16:25:05 -0300 Subject: [PATCH 05/14] Fix tests --- lib/util/http/queue.spec.ts | 10 +++++----- lib/util/http/throttle.spec.ts | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/util/http/queue.spec.ts b/lib/util/http/queue.spec.ts index 69069522393daa..ac8f12c0d5e9a7 100644 --- a/lib/util/http/queue.spec.ts +++ b/lib/util/http/queue.spec.ts @@ -12,15 +12,15 @@ describe('util/http/queue', () => { }); it('returns null for invalid URL', () => { - expect(getQueue('$#@!')).toBeNull(); + expect(getQueue('$#@!', null)).toBeNull(); }); it('returns queue for valid url', () => { - const q1a = getQueue('https://example.com'); - const q1b = getQueue('https://example.com'); + const q1a = getQueue('https://example.com', null); + const q1b = getQueue('https://example.com', null); - const q2a = getQueue('https://example.com:8080'); - const q2b = getQueue('https://example.com:8080'); + const q2a = getQueue('https://example.com:8080', null); + const q2b = getQueue('https://example.com:8080', null); expect(q1a).not.toBeNull(); expect(q1a).toBe(q1b); diff --git a/lib/util/http/throttle.spec.ts b/lib/util/http/throttle.spec.ts index 5163f23defb177..d0e3a2f1ce4cb8 100644 --- a/lib/util/http/throttle.spec.ts +++ b/lib/util/http/throttle.spec.ts @@ -12,15 +12,15 @@ describe('util/http/throttle', () => { }); it('returns null for invalid URL', () => { - expect(getThrottle('$#@!')).toBeNull(); + expect(getThrottle('$#@!', null)).toBeNull(); }); it('returns throttle for valid url', () => { - const t1a = getThrottle('https://example.com'); - const t1b = getThrottle('https://example.com'); + const t1a = getThrottle('https://example.com', null); + const t1b = getThrottle('https://example.com', null); - const t2a = getThrottle('https://example.com:8080'); - const t2b = getThrottle('https://example.com:8080'); + const t2a = getThrottle('https://example.com:8080', null); + const t2b = getThrottle('https://example.com:8080', null); expect(t1a).not.toBeNull(); expect(t1a).toBe(t1b); From 449ed80a5e6be28edc655fd6478e2b7d011b31b5 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 28 Feb 2024 16:47:46 -0300 Subject: [PATCH 06/14] Add more tests --- lib/util/http/host-rules.spec.ts | 51 +++++++++++++++++++++++++++++++- lib/util/http/host-rules.ts | 2 -- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/lib/util/http/host-rules.spec.ts b/lib/util/http/host-rules.spec.ts index 2b452132838578..60ddd0b33bf7f7 100644 --- a/lib/util/http/host-rules.spec.ts +++ b/lib/util/http/host-rules.spec.ts @@ -3,7 +3,12 @@ import { bootstrap } from '../../proxy'; import type { HostRule } from '../../types'; import * as hostRules from '../host-rules'; import { dnsLookup } from './dns'; -import { applyHostRule, findMatchingRule } from './host-rules'; +import { + applyHostRule, + findMatchingRule, + getConcurrentRequestsLimit, + getThrottleIntervalMs, +} from './host-rules'; import type { GotOptions } from './types'; const url = 'https://github.com'; @@ -560,4 +565,48 @@ describe('util/http/host-rules', () => { }, }); }); + + describe('getConcurrentRequestsLimit', () => { + it('returns default value for undefined rule', () => { + expect(getConcurrentRequestsLimit('https://example.com', 42)).toBe(42); + }); + + it('returns null for -1', () => { + hostRules.add({ + matchHost: 'https://example.com', + concurrentRequestLimit: -1, + }); + expect(getConcurrentRequestsLimit('https://example.com', 42)).toBeNull(); + }); + + it('returns positive limit', () => { + hostRules.add({ + matchHost: 'https://example.com', + concurrentRequestLimit: 143, + }); + expect(getConcurrentRequestsLimit('https://example.com', 42)).toBe(143); + }); + }); + + describe('getThrottleIntervalMs', () => { + it('returns default value for undefined rule', () => { + expect(getThrottleIntervalMs('https://example.com', 42)).toBe(24); // 1000 / 42 + }); + + it('returns null for -1', () => { + hostRules.add({ + matchHost: 'https://example.com', + maxRequestsPerSecond: -1, + }); + expect(getThrottleIntervalMs('https://example.com', 42)).toBeNull(); + }); + + it('returns positive limit', () => { + hostRules.add({ + matchHost: 'https://example.com', + maxRequestsPerSecond: 143, + }); + expect(getThrottleIntervalMs('https://example.com', 42)).toBe(7); // 1000 / 143 + }); + }); }); diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index 6592f9f89db119..54387c4ec3bb86 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -231,7 +231,6 @@ export function getConcurrentRequestsLimit( return concurrentRequestLimit; } - // -1 for reset return null; } @@ -249,6 +248,5 @@ export function getThrottleIntervalMs( return Math.ceil(1000 / maxRequestsPerSecond); } - // -1 for reset return null; } From c745686a99e26a658b7fca2cfc7788f54220aaa2 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 4 Mar 2024 12:29:20 -0300 Subject: [PATCH 07/14] Update documentation --- lib/config/options/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 4b441f962d4770..694c294a0deb99 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2398,17 +2398,17 @@ const options: RenovateOptions[] = [ }, { name: 'concurrentRequestLimit', - description: 'Limit concurrent requests per host.', + description: + 'Limit concurrent requests per host. Set to `-1` for no limit. Default value is 5 concurrent requests.', type: 'integer', stage: 'repository', parents: ['hostRules'], - default: null, cli: false, env: false, }, { name: 'maxRequestsPerSecond', - description: 'Limit requests rate per host.', + description: 'Limit requests rate per host. Set to `-1` for no limit. Default value is 5 requests per second.', type: 'integer', stage: 'repository', parents: ['hostRules'], From 7e03742e9a38e79df0e37732482239fa0b407d56 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 4 Mar 2024 12:36:52 -0300 Subject: [PATCH 08/14] Fix prettier --- lib/config/options/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 00f0bbc7988acb..d7cd2054adbda0 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2426,7 +2426,8 @@ const options: RenovateOptions[] = [ }, { name: 'maxRequestsPerSecond', - description: 'Limit requests rate per host. Set to `-1` for no limit. Default value is 5 requests per second.', + description: + 'Limit requests rate per host. Set to `-1` for no limit. Default value is 5 requests per second.', type: 'integer', stage: 'repository', parents: ['hostRules'], From bb1e864fe934bdadddcef9c07f1ce3b55dbfb93e Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 5 Mar 2024 11:17:51 -0300 Subject: [PATCH 09/14] Add default values for documenting purposes --- lib/config/options/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index f750215297c8a7..2bcbeddc351184 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2419,6 +2419,7 @@ const options: RenovateOptions[] = [ description: 'Limit concurrent requests per host. Set to `-1` for no limit. Default value is 5 concurrent requests.', type: 'integer', + default: 5, stage: 'repository', parents: ['hostRules'], cli: false, @@ -2429,9 +2430,9 @@ const options: RenovateOptions[] = [ description: 'Limit requests rate per host. Set to `-1` for no limit. Default value is 5 requests per second.', type: 'integer', + default: 5, stage: 'repository', parents: ['hostRules'], - default: 0, cli: false, env: false, }, From 4e99f8fae3c208232e2b7003b4b256d7c36056c1 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 5 Mar 2024 11:18:52 -0300 Subject: [PATCH 10/14] Change default concurrency to 10 --- lib/config/options/index.ts | 4 ++-- lib/util/http/index.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 2bcbeddc351184..19e49dd1f9a5a5 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2417,9 +2417,9 @@ const options: RenovateOptions[] = [ { name: 'concurrentRequestLimit', description: - 'Limit concurrent requests per host. Set to `-1` for no limit. Default value is 5 concurrent requests.', + 'Limit concurrent requests per host. Set to `-1` for no limit. Default value is 10 concurrent requests.', type: 'integer', - default: 5, + default: 10, stage: 'repository', parents: ['hostRules'], cli: false, diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 920620fb1086aa..fc7fe5e715c29e 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -130,7 +130,7 @@ export class Http { private static defaultMaxRequestsPerSecond: number | null = null; static setDefaultLimits(): void { - Http.defaultConcurrentRequestLimit = 5; + Http.defaultConcurrentRequestLimit = 10; Http.defaultMaxRequestsPerSecond = 5; } From 2e07ddade5c10277c31473f0413715f99de45a69 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 5 Mar 2024 11:22:17 -0300 Subject: [PATCH 11/14] Revert "Change default concurrency to 10" This reverts commit 4e99f8fae3c208232e2b7003b4b256d7c36056c1. --- lib/config/options/index.ts | 4 ++-- lib/util/http/index.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 19e49dd1f9a5a5..2bcbeddc351184 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2417,9 +2417,9 @@ const options: RenovateOptions[] = [ { name: 'concurrentRequestLimit', description: - 'Limit concurrent requests per host. Set to `-1` for no limit. Default value is 10 concurrent requests.', + 'Limit concurrent requests per host. Set to `-1` for no limit. Default value is 5 concurrent requests.', type: 'integer', - default: 10, + default: 5, stage: 'repository', parents: ['hostRules'], cli: false, diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index fc7fe5e715c29e..920620fb1086aa 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -130,7 +130,7 @@ export class Http { private static defaultMaxRequestsPerSecond: number | null = null; static setDefaultLimits(): void { - Http.defaultConcurrentRequestLimit = 10; + Http.defaultConcurrentRequestLimit = 5; Http.defaultMaxRequestsPerSecond = 5; } From 2fd56d3c10474bd41405c297f53436c9611baa6f Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 5 Mar 2024 11:23:23 -0300 Subject: [PATCH 12/14] Shorten descriptions --- lib/config/options/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 2bcbeddc351184..73966ba3f7847c 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2417,7 +2417,7 @@ const options: RenovateOptions[] = [ { name: 'concurrentRequestLimit', description: - 'Limit concurrent requests per host. Set to `-1` for no limit. Default value is 5 concurrent requests.', + 'Limit concurrent requests per host. Set to `-1` for no limit.', type: 'integer', default: 5, stage: 'repository', @@ -2427,8 +2427,7 @@ const options: RenovateOptions[] = [ }, { name: 'maxRequestsPerSecond', - description: - 'Limit requests rate per host. Set to `-1` for no limit. Default value is 5 requests per second.', + description: 'Limit requests rate per host. Set to `-1` for no limit.', type: 'integer', default: 5, stage: 'repository', From 84585a62073fb5c9795432ed12138e1eb7d2fe7a Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 5 Mar 2024 11:27:32 -0300 Subject: [PATCH 13/14] Change tests to use 0 instead of -1 --- lib/config/options/index.ts | 4 ++-- lib/util/http/host-rules.spec.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 73966ba3f7847c..9e2898d96bf3a7 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2417,7 +2417,7 @@ const options: RenovateOptions[] = [ { name: 'concurrentRequestLimit', description: - 'Limit concurrent requests per host. Set to `-1` for no limit.', + 'Limit concurrent requests per host. Set to 0 for no limit.', type: 'integer', default: 5, stage: 'repository', @@ -2427,7 +2427,7 @@ const options: RenovateOptions[] = [ }, { name: 'maxRequestsPerSecond', - description: 'Limit requests rate per host. Set to `-1` for no limit.', + description: 'Limit requests rate per host. Set to 0 for no limit.', type: 'integer', default: 5, stage: 'repository', diff --git a/lib/util/http/host-rules.spec.ts b/lib/util/http/host-rules.spec.ts index 60ddd0b33bf7f7..d558558629995a 100644 --- a/lib/util/http/host-rules.spec.ts +++ b/lib/util/http/host-rules.spec.ts @@ -571,10 +571,10 @@ describe('util/http/host-rules', () => { expect(getConcurrentRequestsLimit('https://example.com', 42)).toBe(42); }); - it('returns null for -1', () => { + it('returns null for 0', () => { hostRules.add({ matchHost: 'https://example.com', - concurrentRequestLimit: -1, + concurrentRequestLimit: 0, }); expect(getConcurrentRequestsLimit('https://example.com', 42)).toBeNull(); }); @@ -593,10 +593,10 @@ describe('util/http/host-rules', () => { expect(getThrottleIntervalMs('https://example.com', 42)).toBe(24); // 1000 / 42 }); - it('returns null for -1', () => { + it('returns null for 0', () => { hostRules.add({ matchHost: 'https://example.com', - maxRequestsPerSecond: -1, + maxRequestsPerSecond: 0, }); expect(getThrottleIntervalMs('https://example.com', 42)).toBeNull(); }); From bbbac970b0a09ccca70e3f69daea49677126ce6a Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 5 Mar 2024 11:29:10 -0300 Subject: [PATCH 14/14] Fix prettier --- lib/config/options/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 9e2898d96bf3a7..6f9c5cfe1107bc 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2416,8 +2416,7 @@ const options: RenovateOptions[] = [ }, { name: 'concurrentRequestLimit', - description: - 'Limit concurrent requests per host. Set to 0 for no limit.', + description: 'Limit concurrent requests per host. Set to 0 for no limit.', type: 'integer', default: 5, stage: 'repository',