From 06f71346cdbfd434652f5f6e699545f0ce57f40b Mon Sep 17 00:00:00 2001 From: Adam Setch Date: Sat, 20 May 2023 01:43:00 -0400 Subject: [PATCH] feat(bitbucket): add support for pagelen (#22278) --- lib/modules/platform/bitbucket/index.spec.ts | 24 ++++-- lib/util/http/bitbucket.spec.ts | 77 ++++++++++++++++++-- lib/util/http/bitbucket.ts | 63 +++++++--------- 3 files changed, 116 insertions(+), 48 deletions(-) diff --git a/lib/modules/platform/bitbucket/index.spec.ts b/lib/modules/platform/bitbucket/index.spec.ts index 70a93df243c374..4e1cd431905e7f 100644 --- a/lib/modules/platform/bitbucket/index.spec.ts +++ b/lib/modules/platform/bitbucket/index.spec.ts @@ -813,7 +813,9 @@ describe('modules/platform/bitbucket/index', () => { }; const scope = await initRepoMock(); scope - .get('/2.0/repositories/some/repo/effective-default-reviewers') + .get( + '/2.0/repositories/some/repo/effective-default-reviewers?pagelen=100' + ) .reply(200, { values: [projectReviewer, repoReviewer], }) @@ -855,7 +857,9 @@ describe('modules/platform/bitbucket/index', () => { }; const scope = await initRepoMock(); scope - .get('/2.0/repositories/some/repo/effective-default-reviewers') + .get( + '/2.0/repositories/some/repo/effective-default-reviewers?pagelen=100' + ) .reply(200, { values: [ activeReviewerWithinWorkspace, @@ -924,7 +928,9 @@ describe('modules/platform/bitbucket/index', () => { }; const scope = await initRepoMock(); scope - .get('/2.0/repositories/some/repo/effective-default-reviewers') + .get( + '/2.0/repositories/some/repo/effective-default-reviewers?pagelen=100' + ) .reply(200, { values: [memberReviewer, notMemberReviewer], }) @@ -973,7 +979,9 @@ describe('modules/platform/bitbucket/index', () => { }; const scope = await initRepoMock(); scope - .get('/2.0/repositories/some/repo/effective-default-reviewers') + .get( + '/2.0/repositories/some/repo/effective-default-reviewers?pagelen=100' + ) .reply(200, { values: [reviewer], }) @@ -1017,7 +1025,9 @@ describe('modules/platform/bitbucket/index', () => { const scope = await initRepoMock(); scope - .get('/2.0/repositories/some/repo/effective-default-reviewers') + .get( + '/2.0/repositories/some/repo/effective-default-reviewers?pagelen=100' + ) .reply(200, { values: [reviewer], }) @@ -1054,7 +1064,9 @@ describe('modules/platform/bitbucket/index', () => { const scope = await initRepoMock(); scope - .get('/2.0/repositories/some/repo/effective-default-reviewers') + .get( + '/2.0/repositories/some/repo/effective-default-reviewers?pagelen=100' + ) .reply(200, { values: [reviewer], }) diff --git a/lib/util/http/bitbucket.spec.ts b/lib/util/http/bitbucket.spec.ts index 7f74909d1c8175..ea6c9e6624def1 100644 --- a/lib/util/http/bitbucket.spec.ts +++ b/lib/util/http/bitbucket.spec.ts @@ -56,27 +56,90 @@ describe('util/http/bitbucket', () => { }); }); - it('paginates', async () => { + it('paginates: adds default pagelen if non is present', async () => { httpMock .scope(baseUrl) - .get('/some-url') + .get('/some-url?foo=bar&pagelen=100') .reply(200, { values: ['a'], page: '1', - next: `${baseUrl}/some-url?page=2`, + next: `${baseUrl}/some-url?foo=bar&pagelen=100&page=2`, }) - .get('/some-url?page=2') + .get('/some-url?foo=bar&pagelen=100&page=2') .reply(200, { values: ['b', 'c'], page: '2', - next: `${baseUrl}/some-url?page=3`, + next: `${baseUrl}/some-url?foo=bar&pagelen=100&page=3`, }) - .get('/some-url?page=3') + .get('/some-url?foo=bar&pagelen=100&page=3') .reply(200, { values: ['d'], page: '3', }); - const res = await api.getJson('some-url', { paginate: true }); + const res = await api.getJson('/some-url?foo=bar', { paginate: true }); + expect(res.body).toEqual({ + page: '1', + pagelen: 4, + size: 4, + values: ['a', 'b', 'c', 'd'], + next: undefined, + }); + }); + + it('paginates: respects pagelen if already set in path', async () => { + httpMock + .scope(baseUrl) + .get('/some-url?pagelen=10') + .reply(200, { + values: ['a'], + page: '1', + next: `${baseUrl}/some-url?pagelen=10&page=2`, + }) + .get('/some-url?pagelen=10&page=2') + .reply(200, { + values: ['b', 'c'], + page: '2', + next: `${baseUrl}/some-url?pagelen=10&page=3`, + }) + .get('/some-url?pagelen=10&page=3') + .reply(200, { + values: ['d'], + page: '3', + }); + const res = await api.getJson('some-url?pagelen=10', { paginate: true }); + expect(res.body).toEqual({ + page: '1', + pagelen: 4, + size: 4, + values: ['a', 'b', 'c', 'd'], + next: undefined, + }); + }); + + it('paginates: respects pagelen if set in options', async () => { + httpMock + .scope(baseUrl) + .get('/some-url?pagelen=20') + .reply(200, { + values: ['a'], + page: '1', + next: `${baseUrl}/some-url?pagelen=20&page=2`, + }) + .get('/some-url?pagelen=20&page=2') + .reply(200, { + values: ['b', 'c'], + page: '2', + next: `${baseUrl}/some-url?pagelen=20&page=3`, + }) + .get('/some-url?pagelen=20&page=3') + .reply(200, { + values: ['d'], + page: '3', + }); + const res = await api.getJson('some-url', { + paginate: true, + pagelen: 20, + }); expect(res.body).toEqual({ page: '1', pagelen: 4, diff --git a/lib/util/http/bitbucket.ts b/lib/util/http/bitbucket.ts index 15bea99b04860a..0d4a51c28a9b50 100644 --- a/lib/util/http/bitbucket.ts +++ b/lib/util/http/bitbucket.ts @@ -1,9 +1,13 @@ import is from '@sindresorhus/is'; +import { logger } from '../../logger'; import type { PagedResult } from '../../modules/platform/bitbucket/types'; import { parseUrl, resolveBaseUrl } from '../url'; import type { HttpOptions, HttpResponse } from './types'; import { Http } from '.'; +const MAX_PAGES = 100; +const MAX_PAGELEN = 100; + let baseUrl = 'https://api.bitbucket.org/'; export const setBaseUrl = (url: string): void => { @@ -12,6 +16,7 @@ export const setBaseUrl = (url: string): void => { export interface BitbucketHttpOptions extends HttpOptions { paginate?: boolean; + pagelen?: number; } export class BitbucketHttp extends Http { @@ -25,62 +30,50 @@ export class BitbucketHttp extends Http { ): Promise> { const opts = { baseUrl, ...options }; - const result = await super.request(path, opts); + const resolvedURL = parseUrl(resolveBaseUrl(baseUrl, path)); - if (opts.paginate && isPagedResult(result.body)) { - const resultBody = result.body as PagedResult; + // istanbul ignore if: this should never happen + if (is.nullOrUndefined(resolvedURL)) { + logger.error(`Bitbucket: cannot parse path ${path}`); + throw new Error(`Bitbucket: cannot parse path ${path}`); + } - let nextPage = getPageFromURL(resultBody.next); + if (opts.paginate && !hasPagelen(resolvedURL)) { + const pagelen = opts.pagelen ?? MAX_PAGELEN; + resolvedURL.searchParams.set('pagelen', pagelen.toString()); + } - while (is.nonEmptyString(nextPage)) { - const nextPath = getNextPagePath(path, nextPage); + const result = await super.request(resolvedURL.toString(), opts); - // istanbul ignore if - if (is.nullOrUndefined(nextPath)) { - break; - } + if (opts.paginate && isPagedResult(result.body)) { + const resultBody = result.body as PagedResult; + let page = 1; + let nextURL = resultBody.next; + while (is.nonEmptyString(nextURL) && page <= MAX_PAGES) { const nextResult = await super.request>( - nextPath, + nextURL, options ); resultBody.values.push(...nextResult.body.values); - nextPage = getPageFromURL(nextResult.body?.next); + nextURL = nextResult.body?.next; + page += 1; } // Override other page-related attributes resultBody.pagelen = resultBody.values.length; - resultBody.size = resultBody.values.length; - resultBody.next = undefined; + resultBody.size = page > MAX_PAGES ? undefined : resultBody.values.length; + resultBody.next = page > MAX_PAGES ? undefined : nextURL; } return result; } } -function getPageFromURL(url: string | undefined): string | null { - const resolvedURL = parseUrl(url); - - if (is.nullOrUndefined(resolvedURL)) { - return null; - } - - return resolvedURL.searchParams.get('page'); -} - -function getNextPagePath(path: string, nextPage: string): string | null { - const resolvedURL = parseUrl(resolveBaseUrl(baseUrl, path)); - - // istanbul ignore if - if (is.nullOrUndefined(resolvedURL)) { - return null; - } - - resolvedURL.searchParams.set('page', nextPage); - - return resolvedURL.toString(); +function hasPagelen(url: URL): boolean { + return !is.nullOrUndefined(url.searchParams.get('pagelen')); } function isPagedResult(obj: any): obj is PagedResult {