Limit number of requests in pagination (#1181)

* feat: limit number of requests in pagination

* style: fix lint

* fix: increase default to 10000

* fix: use lower than comparator

* docs: add note about retries

* docs: fix typo

* docs: update readme

Author: jaulz

readme.md

@@ -760,6 +760,15 @@ Default: `Infinity`
 
 The maximum amount of items that should be emitted.
 
+###### pagination.requestLimit
+
+Type: `number`\
+Default: `10000`
+
+The maximum amount of request that should be triggered. [Retries on failure](#retry) are not counted towards this limit.
+
+For example, it can be helpful during development to avoid an infinite number of requests.
+
 ##### localAddress
 
 Type: `string`

source/as-promise/types.ts

@@ -77,6 +77,7 @@ export interface PaginationOptions<T> {
 		paginate?: (response: Response, allItems: T[], currentItems: T[]) => Options | false;
 		shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean;
 		countLimit?: number;
+		requestLimit?: number;
 	};
 }
 

source/create.ts

@@ -206,7 +206,8 @@ const create = (defaults: InstanceDefaults): Got => {
 
 		const all: T[] = [];
 
-		while (true) {
+		let numberOfRequests = 0;
+		while (numberOfRequests < pagination.requestLimit) {
 			// TODO: Throw when result is not an instance of Response
 			// eslint-disable-next-line no-await-in-loop
 			const result = (await got('', normalizedOptions)) as Response;
@@ -241,6 +242,8 @@ const create = (defaults: InstanceDefaults): Got => {
 			if (optionsToMerge !== undefined) {
 				normalizedOptions = normalizeArguments(undefined, optionsToMerge, normalizedOptions);
 			}
+
+			numberOfRequests++;
 		}
 	}) as GotPaginate;
 

source/index.ts

@@ -108,7 +108,8 @@ const defaults: InstanceDefaults = {
 			},
 			filter: () => true,
 			shouldContinue: () => true,
-			countLimit: Infinity
+			countLimit: Infinity,
+			requestLimit: 10000
 		}
 	},
 	handlers: [defaultHandler],

test/pagination.ts

@@ -328,3 +328,21 @@ test.failing('allowGetBody sends json payload with .paginate()', withBodyParsing
 
 	t.deepEqual(result.value, [1, 2, 3]);
 });
+
+test('`requestLimit` works', withServer, async (t, server, got) => {
+	attachHandler(server, 2);
+
+	const options = {
+		pagination: {
+			requestLimit: 1
+		}
+	};
+
+	const results: number[] = [];
+
+	for await (const item of got.paginate<number>(options)) {
+		results.push(item);
+	}
+
+	t.deepEqual(results, [1]);
+});