Pass allItems and currentItems to _pagination.paginate() (#1100)

Co-authored-by: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Co-authored-by: Sindre Sorhus <sindresorhus@gmail.com>

Author: 3 people

readme.md

@@ -682,23 +682,64 @@ A function that transform [`Response`](#response) into an array of items. This i
 Type: `Function`\
 Default: [`Link` header logic](source/index.ts)
 
-A function that returns an object representing Got options pointing to the next page. If there are no more pages, `false` should be returned.
+The function takes three arguments:
+- `response` - The current response object.
+- `allItems` - An array of the emitted items.
+- `currentItems` - Items from the current response.
+
+It should return an object representing Got options pointing to the next page. If there are no more pages, `false` should be returned.
+
+For example, if you want to stop when the response contains less items than expected, you can use something like this:
+
+```js
+const got = require('got');
+
+(async () => {
+	const limit = 10;
+
+	const items = got.paginate('https://example.com/items', {
+		searchParams: {
+			limit,
+			offset: 0
+		},
+		_pagination: {
+			paginate: (response, allItems, currentItems) => {
+				const previousSearchParams = response.request.options.searchParams;
+				const {offset: previousOffset} = previousSearchParams;
+
+				if (currentItems.length < limit) {
+					return false;
+				}
+
+				return {
+					searchParams: {
+						...previousSearchParams,
+						offset: previousOffset + limit,
+					}
+				};
+			}
+		}
+	});
+
+	console.log('Items from all pages:', items);
+})();
+```
 
 ###### \_pagination.filter
 
 Type: `Function`\
-Default: `(item, allItems) => true`
+Default: `(item, allItems, currentItems) => true`
 
 Checks whether the item should be emitted or not.
 
 ###### \_pagination.shouldContinue
 
 Type: `Function`\
-Default: `(item, allItems) => true`
+Default: `(item, allItems, currentItems) => true`
 
 Checks whether the pagination should continue.
 
-For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems) => !item.flag`. If you want to stop **after** emitting the entry, you should use `(item, allItems) => allItems.some(entry => entry.flag)` instead.
+For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems, currentItems) => !item.flag`. If you want to stop **after** emitting the entry, you should use `(item, allItems, currentItems) => allItems.some(entry => entry.flag)` instead.
 
 ###### \_pagination.countLimit
 

source/create.ts

@@ -216,24 +216,26 @@ const create = (defaults: Defaults): Got => {
 
 			// eslint-disable-next-line no-await-in-loop
 			const parsed = await pagination.transform!(result);
+			const current: T[] = [];
 
 			for (const item of parsed) {
-				if (pagination.filter!(item, all)) {
-					if (!pagination.shouldContinue!(item, all)) {
+				if (pagination.filter!(item, all, current)) {
+					if (!pagination.shouldContinue!(item, all, current)) {
 						return;
 					}
 
 					yield item;
 
 					all.push(item as T);
+					current.push(item as T);
 
 					if (all.length === pagination.countLimit) {
 						return;
 					}
 				}
 			}
 
-			const optionsToMerge = pagination.paginate!(result);
+			const optionsToMerge = pagination.paginate!(result, all, current);
 
 			if (optionsToMerge === false) {
 				return;

source/types.ts

@@ -163,9 +163,9 @@ export type DefaultOptions = Merge<
 export interface PaginationOptions<T> {
 	_pagination?: {
 		transform?: (response: Response) => Promise<T[]> | T[];
-		filter?: (item: T, allItems: T[]) => boolean;
-		paginate?: (response: Response) => Options | false;
-		shouldContinue?: (item: T, allItems: T[]) => boolean;
+		filter?: (item: T, allItems: T[], currentItems: T[]) => boolean;
+		paginate?: (response: Response, allItems: T[], currentItems: T[]) => Options | false;
+		shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean;
 		countLimit?: number;
 	};
 }

test/pagination.ts

@@ -82,7 +82,12 @@ test('filters elements', withServer, async (t, server, got) => {
 
 	const result = await got.paginate.all({
 		_pagination: {
-			filter: element => element !== 2
+			filter: (element, allItems, currentItems) => {
+				t.true(Array.isArray(allItems));
+				t.true(Array.isArray(currentItems));
+
+				return element !== 2;
+			}
 		}
 	});
 
@@ -131,6 +136,42 @@ test('custom paginate function', withServer, async (t, server, got) => {
 	t.deepEqual(result, [1, 3]);
 });
 
+test('custom paginate function using allItems', withServer, async (t, server, got) => {
+	attachHandler(server, 3);
+
+	const result = await got.paginate.all({
+		_pagination: {
+			paginate: (_response, allItems) => {
+				if (allItems.length === 2) {
+					return false;
+				}
+
+				return {path: '/?page=3'};
+			}
+		}
+	});
+
+	t.deepEqual(result, [1, 3]);
+});
+
+test('custom paginate function using currentItems', withServer, async (t, server, got) => {
+	attachHandler(server, 3);
+
+	const result = await got.paginate.all({
+		_pagination: {
+			paginate: (_response, _allItems, currentItems) => {
+				if (currentItems[0] === 3) {
+					return false;
+				}
+
+				return {path: '/?page=3'};
+			}
+		}
+	});
+
+	t.deepEqual(result, [1, 3]);
+});
+
 test('iterator works', withServer, async (t, server, got) => {
 	attachHandler(server, 5);
 
@@ -148,7 +189,12 @@ test('`shouldContinue` works', withServer, async (t, server, got) => {
 
 	const options = {
 		_pagination: {
-			shouldContinue: () => false
+			shouldContinue: (_element: unknown, allItems: unknown[], currentItems: unknown[]) => {
+				t.true(Array.isArray(allItems));
+				t.true(Array.isArray(currentItems));
+
+				return false;
+			}
 		}
 	};