From 1cddd5264696d3eda6b4deb98c45437cb4a2e1db Mon Sep 17 00:00:00 2001 From: Julian Hundeloh Date: Mon, 9 Mar 2020 13:42:22 +0100 Subject: [PATCH] Pass `allItems` and `currentItems` to `_pagination.paginate()` (#1100) Co-authored-by: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Co-authored-by: Sindre Sorhus --- readme.md | 49 +++++++++++++++++++++++++++++++++++++++++---- source/create.ts | 8 +++++--- source/types.ts | 6 +++--- test/pagination.ts | 50 ++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 101 insertions(+), 12 deletions(-) diff --git a/readme.md b/readme.md index 14123188b..4b68234d6 100644 --- a/readme.md +++ b/readme.md @@ -682,23 +682,64 @@ A function that transform [`Response`](#response) into an array of items. This i Type: `Function`\ Default: [`Link` header logic](source/index.ts) -A function that returns an object representing Got options pointing to the next page. If there are no more pages, `false` should be returned. +The function takes three arguments: +- `response` - The current response object. +- `allItems` - An array of the emitted items. +- `currentItems` - Items from the current response. + +It should return an object representing Got options pointing to the next page. If there are no more pages, `false` should be returned. + +For example, if you want to stop when the response contains less items than expected, you can use something like this: + +```js +const got = require('got'); + +(async () => { + const limit = 10; + + const items = got.paginate('https://example.com/items', { + searchParams: { + limit, + offset: 0 + }, + _pagination: { + paginate: (response, allItems, currentItems) => { + const previousSearchParams = response.request.options.searchParams; + const {offset: previousOffset} = previousSearchParams; + + if (currentItems.length < limit) { + return false; + } + + return { + searchParams: { + ...previousSearchParams, + offset: previousOffset + limit, + } + }; + } + } + }); + + console.log('Items from all pages:', items); +})(); +``` ###### \_pagination.filter Type: `Function`\ -Default: `(item, allItems) => true` +Default: `(item, allItems, currentItems) => true` Checks whether the item should be emitted or not. ###### \_pagination.shouldContinue Type: `Function`\ -Default: `(item, allItems) => true` +Default: `(item, allItems, currentItems) => true` Checks whether the pagination should continue. -For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems) => !item.flag`. If you want to stop **after** emitting the entry, you should use `(item, allItems) => allItems.some(entry => entry.flag)` instead. +For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems, currentItems) => !item.flag`. If you want to stop **after** emitting the entry, you should use `(item, allItems, currentItems) => allItems.some(entry => entry.flag)` instead. ###### \_pagination.countLimit diff --git a/source/create.ts b/source/create.ts index e59a39ee6..ae4b46eb7 100644 --- a/source/create.ts +++ b/source/create.ts @@ -216,16 +216,18 @@ const create = (defaults: Defaults): Got => { // eslint-disable-next-line no-await-in-loop const parsed = await pagination.transform!(result); + const current: T[] = []; for (const item of parsed) { - if (pagination.filter!(item, all)) { - if (!pagination.shouldContinue!(item, all)) { + if (pagination.filter!(item, all, current)) { + if (!pagination.shouldContinue!(item, all, current)) { return; } yield item; all.push(item as T); + current.push(item as T); if (all.length === pagination.countLimit) { return; @@ -233,7 +235,7 @@ const create = (defaults: Defaults): Got => { } } - const optionsToMerge = pagination.paginate!(result); + const optionsToMerge = pagination.paginate!(result, all, current); if (optionsToMerge === false) { return; diff --git a/source/types.ts b/source/types.ts index 4be4eccdf..e9803f872 100644 --- a/source/types.ts +++ b/source/types.ts @@ -163,9 +163,9 @@ export type DefaultOptions = Merge< export interface PaginationOptions { _pagination?: { transform?: (response: Response) => Promise | T[]; - filter?: (item: T, allItems: T[]) => boolean; - paginate?: (response: Response) => Options | false; - shouldContinue?: (item: T, allItems: T[]) => boolean; + filter?: (item: T, allItems: T[], currentItems: T[]) => boolean; + paginate?: (response: Response, allItems: T[], currentItems: T[]) => Options | false; + shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean; countLimit?: number; }; } diff --git a/test/pagination.ts b/test/pagination.ts index 5d457e8f3..3446e0fa2 100644 --- a/test/pagination.ts +++ b/test/pagination.ts @@ -82,7 +82,12 @@ test('filters elements', withServer, async (t, server, got) => { const result = await got.paginate.all({ _pagination: { - filter: element => element !== 2 + filter: (element, allItems, currentItems) => { + t.true(Array.isArray(allItems)); + t.true(Array.isArray(currentItems)); + + return element !== 2; + } } }); @@ -131,6 +136,42 @@ test('custom paginate function', withServer, async (t, server, got) => { t.deepEqual(result, [1, 3]); }); +test('custom paginate function using allItems', withServer, async (t, server, got) => { + attachHandler(server, 3); + + const result = await got.paginate.all({ + _pagination: { + paginate: (_response, allItems) => { + if (allItems.length === 2) { + return false; + } + + return {path: '/?page=3'}; + } + } + }); + + t.deepEqual(result, [1, 3]); +}); + +test('custom paginate function using currentItems', withServer, async (t, server, got) => { + attachHandler(server, 3); + + const result = await got.paginate.all({ + _pagination: { + paginate: (_response, _allItems, currentItems) => { + if (currentItems[0] === 3) { + return false; + } + + return {path: '/?page=3'}; + } + } + }); + + t.deepEqual(result, [1, 3]); +}); + test('iterator works', withServer, async (t, server, got) => { attachHandler(server, 5); @@ -148,7 +189,12 @@ test('`shouldContinue` works', withServer, async (t, server, got) => { const options = { _pagination: { - shouldContinue: () => false + shouldContinue: (_element: unknown, allItems: unknown[], currentItems: unknown[]) => { + t.true(Array.isArray(allItems)); + t.true(Array.isArray(currentItems)); + + return false; + } } };