From 25679872b66f3f7364e53a4f4e1055bb8627220a Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Fri, 28 Feb 2020 20:08:57 +0100 Subject: [PATCH] Rewrite Got --- package.json | 25 +- source/as-promise.ts | 182 -- .../{ => as-promise}/calculate-retry-delay.ts | 18 +- source/as-promise/core.ts | 133 ++ source/as-promise/create-rejection.ts | 13 + source/as-promise/index.ts | 242 +++ source/as-promise/types.ts | 149 ++ source/as-stream.ts | 144 -- source/core/index.ts | 1462 +++++++++++++++++ source/create.ts | 249 ++- source/errors.ts | 125 -- source/get-response.ts | 37 - source/index.ts | 56 +- source/known-hook-events.ts | 117 -- source/normalize-arguments.ts | 536 ------ source/progress.ts | 48 - source/request-as-event-emitter.ts | 345 ---- source/types.ts | 274 --- source/types/mimic-response/index.d.ts | 8 + source/types/reflect/index.d.ts | 5 - source/utils/dynamic-require.ts | 3 - source/utils/merge.ts | 32 - source/utils/options-to-url.ts | 113 -- source/utils/proxy-events.ts | 22 + source/utils/supports-brotli.ts | 3 - source/utils/timed-out.ts | 17 +- test/agent.ts | 12 +- test/arguments.ts | 107 +- test/cache.ts | 30 +- test/cookies.ts | 2 +- test/create.ts | 24 +- test/error.ts | 46 +- test/gzip.ts | 2 +- test/headers.ts | 26 +- test/helpers/with-server.ts | 4 +- test/hooks.ts | 71 +- test/http.ts | 45 +- test/https.ts | 14 + test/options-to-url.ts | 150 -- test/pagination.ts | 14 +- test/post.ts | 28 +- test/progress.ts | 16 +- test/promise.ts | 5 +- test/redirects.ts | 14 +- test/response-parse.ts | 4 +- test/retry.ts | 7 +- test/stream.ts | 36 +- test/timeout.ts | 34 +- test/unix-socket.ts | 2 +- 49 files changed, 2587 insertions(+), 2464 deletions(-) delete mode 100644 source/as-promise.ts rename source/{ => as-promise}/calculate-retry-delay.ts (72%) create mode 100644 source/as-promise/core.ts create mode 100644 source/as-promise/create-rejection.ts create mode 100644 source/as-promise/index.ts create mode 100644 source/as-promise/types.ts delete mode 100644 source/as-stream.ts create mode 100644 source/core/index.ts delete mode 100644 source/errors.ts delete mode 100644 source/get-response.ts delete mode 100644 source/known-hook-events.ts delete mode 100644 source/normalize-arguments.ts delete mode 100644 source/progress.ts delete mode 100644 source/request-as-event-emitter.ts delete mode 100644 source/types.ts create mode 100644 source/types/mimic-response/index.d.ts delete mode 100644 source/types/reflect/index.d.ts delete mode 100644 source/utils/dynamic-require.ts delete mode 100644 source/utils/merge.ts delete mode 100644 source/utils/options-to-url.ts create mode 100644 source/utils/proxy-events.ts delete mode 100644 source/utils/supports-brotli.ts delete mode 100644 test/options-to-url.ts diff --git a/package.json b/package.json index a6989c1fa..8e5adab0a 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "node": ">=10" }, "scripts": { - "test": "xo && tsc --noEmit && nyc ava", + "test": "xo && tsc --noEmit && nyc --reporter=html --reporter=text ava", "release": "np", "build": "del-cli dist && tsc", "prepare": "npm run build" @@ -33,29 +33,27 @@ "fetch", "net", "network", - "electron", + "gzip", "brotli", "requests", "human-friendly", "axios", - "superagent" + "superagent", + "node-fetch", + "ky" ], "dependencies": { - "@sindresorhus/is": "^2.0.0", + "@sindresorhus/is": "^2.1.0", "@szmarczak/http-timer": "^4.0.0", "@types/cacheable-request": "^6.0.1", - "cacheable-lookup": "^2.0.0", + "cacheable-lookup": "^3.0.0", "cacheable-request": "^7.0.1", "decompress-response": "^5.0.0", - "duplexer3": "^0.1.4", "get-stream": "^5.0.0", + "http2-wrapper": "^1.0.0-beta.4.2", "lowercase-keys": "^2.0.0", - "mimic-response": "^2.1.0", "p-cancelable": "^2.0.0", - "p-event": "^4.0.0", - "responselike": "^2.0.0", - "to-readable-stream": "^2.0.0", - "type-fest": "^0.10.0" + "responselike": "^2.0.0" }, "devDependencies": { "@ava/typescript": "^1.1.1", @@ -83,10 +81,12 @@ "nock": "^12.0.0", "np": "^6.0.0", "nyc": "^15.0.0", + "p-event": "^4.0.0", "proxyquire": "^2.0.1", "sinon": "^8.1.1", "slow-stream": "0.0.4", "tempy": "^0.4.0", + "to-readable-stream": "^2.1.0", "tough-cookie": "^3.0.0", "typescript": "3.7.5", "xo": "^0.26.0" @@ -94,8 +94,7 @@ "types": "dist/source", "sideEffects": false, "browser": { - "decompress-response": false, - "electron": false + "decompress-response": false }, "ava": { "files": [ diff --git a/source/as-promise.ts b/source/as-promise.ts deleted file mode 100644 index 9f4f2feac..000000000 --- a/source/as-promise.ts +++ /dev/null @@ -1,182 +0,0 @@ -import EventEmitter = require('events'); -import getStream = require('get-stream'); -import PCancelable = require('p-cancelable'); -import is from '@sindresorhus/is'; -import {ParseError, ReadError, HTTPError} from './errors'; -import {normalizeArguments, mergeOptions} from './normalize-arguments'; -import requestAsEventEmitter, {proxyEvents} from './request-as-event-emitter'; -import {CancelableRequest, GeneralError, NormalizedOptions, Response} from './types'; - -const parseBody = (body: Buffer, responseType: NormalizedOptions['responseType'], encoding: NormalizedOptions['encoding']): unknown => { - if (responseType === 'json') { - return body.length === 0 ? '' : JSON.parse(body.toString()); - } - - if (responseType === 'buffer') { - return Buffer.from(body); - } - - if (responseType === 'text') { - return body.toString(encoding); - } - - throw new TypeError(`Unknown body type '${responseType as string}'`); -}; - -export function createRejection(error: Error): CancelableRequest { - const promise = Promise.reject(error) as CancelableRequest; - const returnPromise = (): CancelableRequest => promise; - - promise.json = returnPromise; - promise.text = returnPromise; - promise.buffer = returnPromise; - promise.on = returnPromise; - - return promise; -} - -export default function asPromise(options: NormalizedOptions): CancelableRequest { - const proxy = new EventEmitter(); - let body: Buffer; - - const promise = new PCancelable((resolve, reject, onCancel) => { - const emitter = requestAsEventEmitter(options); - onCancel(emitter.abort); - - const emitError = async (error: GeneralError): Promise => { - try { - for (const hook of options.hooks.beforeError) { - // eslint-disable-next-line no-await-in-loop - error = await hook(error); - } - - reject(error); - } catch (error_) { - reject(error_); - } - }; - - emitter.on('response', async (response: Response) => { - proxy.emit('response', response); - - // Download body - try { - body = await getStream.buffer(response, {encoding: 'binary'}); - } catch (error) { - emitError(new ReadError(error, options)); - return; - } - - if (response.req?.aborted) { - // Canceled while downloading - will throw a `CancelError` or `TimeoutError` error - return; - } - - const isOk = (): boolean => { - const {statusCode} = response; - const limitStatusCode = options.followRedirect ? 299 : 399; - - return (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304; - }; - - // Parse body - try { - response.body = parseBody(body, options.responseType, options.encoding); - } catch (error) { - // Fall back to `utf8` - response.body = body.toString(); - - if (isOk()) { - const parseError = new ParseError(error, response, options); - emitError(parseError); - return; - } - } - - try { - for (const [index, hook] of options.hooks.afterResponse.entries()) { - // @ts-ignore TS doesn't notice that CancelableRequest is a Promise - // eslint-disable-next-line no-await-in-loop - response = await hook(response, async (updatedOptions): CancelableRequest => { - const typedOptions = normalizeArguments(mergeOptions(options, { - ...updatedOptions, - retry: { - calculateDelay: () => 0 - }, - throwHttpErrors: false, - resolveBodyOnly: false - })); - - // Remove any further hooks for that request, because we'll call them anyway. - // The loop continues. We don't want duplicates (asPromise recursion). - typedOptions.hooks.afterResponse = options.hooks.afterResponse.slice(0, index); - - for (const hook of options.hooks.beforeRetry) { - // eslint-disable-next-line no-await-in-loop - await hook(typedOptions); - } - - const promise = asPromise(typedOptions); - - onCancel(() => { - promise.catch(() => {}); - promise.cancel(); - }); - - return promise as unknown as CancelableRequest; - }); - } - } catch (error) { - emitError(error); - return; - } - - // Check for HTTP error codes - if (!isOk()) { - const error = new HTTPError(response, options); - - if (emitter.retry(error)) { - return; - } - - if (options.throwHttpErrors) { - emitError(error); - return; - } - } - - resolve(options.resolveBodyOnly ? response.body : response); - }); - - emitter.once('error', reject); - - proxyEvents(proxy, emitter); - }) as CancelableRequest; - - promise.on = (name: string, fn: (...args: any[]) => void) => { - proxy.on(name, fn); - return promise; - }; - - const shortcut = (responseType: NormalizedOptions['responseType']): CancelableRequest => { - // eslint-disable-next-line promise/prefer-await-to-then - const newPromise = promise.then(() => parseBody(body, responseType, options.encoding)); - - Object.defineProperties(newPromise, Object.getOwnPropertyDescriptors(promise)); - - return newPromise as CancelableRequest; - }; - - promise.json = () => { - if (is.undefined(body) && is.undefined(options.headers.accept)) { - options.headers.accept = 'application/json'; - } - - return shortcut('json'); - }; - - promise.buffer = () => shortcut('buffer'); - promise.text = () => shortcut('text'); - - return promise; -} diff --git a/source/calculate-retry-delay.ts b/source/as-promise/calculate-retry-delay.ts similarity index 72% rename from source/calculate-retry-delay.ts rename to source/as-promise/calculate-retry-delay.ts index 0dcb65303..34c9e261f 100644 --- a/source/calculate-retry-delay.ts +++ b/source/as-promise/calculate-retry-delay.ts @@ -1,6 +1,10 @@ -import is from '@sindresorhus/is'; -import {HTTPError, ParseError, MaxRedirectsError} from './errors'; -import {RetryFunction, RetryObject} from './types'; +import { + ParseError, + HTTPError, + MaxRedirectsError, + RetryObject, + RetryFunction +} from './types'; const retryAfterStatusCodes: ReadonlySet = new Set([413, 429, 503]); @@ -14,7 +18,7 @@ const calculateRetryDelay: RetryFunction = ({attemptCount, retryOptions, error}) } const hasMethod = retryOptions.methods.includes(error.options.method); - const hasErrorCode = Reflect.has(error, 'code') && retryOptions.errorCodes.includes(error.code); + const hasErrorCode = retryOptions.errorCodes.includes(error.code!); const hasStatusCode = isErrorWithResponse(error) && retryOptions.statusCodes.includes(error.response.statusCode); if (!hasMethod || (!hasErrorCode && !hasStatusCode)) { return 0; @@ -22,15 +26,15 @@ const calculateRetryDelay: RetryFunction = ({attemptCount, retryOptions, error}) if (isErrorWithResponse(error)) { const {response} = error; - if (response && Reflect.has(response.headers, 'retry-after') && retryAfterStatusCodes.has(response.statusCode)) { + if (response && 'retry-after' in response.headers && retryAfterStatusCodes.has(response.statusCode)) { let after = Number(response.headers['retry-after']); - if (is.nan(after)) { + if (isNaN(after)) { after = Date.parse(response.headers['retry-after']!) - Date.now(); } else { after *= 1000; } - if (after > retryOptions.maxRetryAfter) { + if (retryOptions.maxRetryAfter === undefined || after > retryOptions.maxRetryAfter) { return 0; } diff --git a/source/as-promise/core.ts b/source/as-promise/core.ts new file mode 100644 index 000000000..015169ee9 --- /dev/null +++ b/source/as-promise/core.ts @@ -0,0 +1,133 @@ +import {URL} from 'url'; +import is, {assert} from '@sindresorhus/is'; +import { + Options, + NormalizedOptions, + Defaults, + ResponseType +} from './types'; +import Request, {knownHookEvents, RequestError, HTTPError} from '../core'; + +if (!knownHookEvents.includes('beforeRetry' as any)) { + knownHookEvents.push('beforeRetry' as any, 'afterResponse' as any); +} + +export const knownBodyTypes = ['json', 'buffer', 'text']; + +// @ts-ignore The error is: Not all code paths return a value. +export const parseBody = (body: Buffer, responseType: ResponseType, encoding?: string): unknown => { + if (responseType === 'text') { + return body.toString(encoding); + } + + if (responseType === 'json') { + return body.length === 0 ? '' : JSON.parse(body.toString()); + } + + if (responseType === 'buffer') { + return Buffer.from(body); + } + + if (!knownBodyTypes.includes(responseType)) { + throw new TypeError(`Unknown body type '${responseType as string}'`); + } +}; + +export default class PromisableRequest extends Request { + ['constructor']: typeof PromisableRequest; + declare options: NormalizedOptions; + declare _throwHttpErrors: boolean; + + static normalizeArguments(url?: string | URL, nonNormalizedOptions?: Options, defaults?: Defaults): NormalizedOptions { + const options = super.normalizeArguments(url, nonNormalizedOptions, defaults) as NormalizedOptions; + + assert.any([is.boolean, is.undefined], options.resolveBodyOnly); + assert.any([is.boolean, is.undefined], options.methodRewriting); + assert.any([is.boolean, is.undefined], options.isStream); + + options.resolveBodyOnly = Boolean(options.resolveBodyOnly); + options.methodRewriting = Boolean(options.methodRewriting); + options.isStream = Boolean(options.isStream); + + // `options.retry` + const {retry} = options; + + if (defaults) { + options.retry = {...defaults.retry}; + } else { + options.retry = { + calculateDelay: retryObject => retryObject.computedValue, + limit: 0, + methods: [], + statusCodes: [], + errorCodes: [], + maxRetryAfter: undefined + }; + } + + if (is.object(retry)) { + options.retry = { + ...options.retry, + ...retry + }; + } else if (is.number(retry)) { + options.retry.limit = retry; + } + + if (is.undefined(options.retry.maxRetryAfter)) { + options.retry.maxRetryAfter = Math.min( + ...[options.timeout.request, options.timeout.connect].filter(is.number) + ); + } + + // `options._pagination` + if (is.object(options._pagination)) { + if (defaults) { + (options as Options)._pagination = { + ...defaults._pagination, + ...options._pagination + }; + } + + const {_pagination: pagination} = options; + + if (!is.function_(pagination.transform)) { + throw new Error('`options._pagination.transform` must be implemented'); + } + + if (!is.function_(pagination.shouldContinue)) { + throw new Error('`options._pagination.shouldContinue` must be implemented'); + } + + if (!is.function_(pagination.filter)) { + throw new TypeError('`options._pagination.filter` must be implemented'); + } + + if (!is.function_(pagination.paginate)) { + throw new Error('`options._pagination.paginate` must be implemented'); + } + } + + return options; + } + + async _beforeError(error: RequestError): Promise { + const isHTTPError = error instanceof HTTPError; + + try { + for (const hook of this.options.hooks.beforeError) { + // eslint-disable-next-line no-await-in-loop + error = await hook(error); + } + } catch (error_) { + this.destroy(error_); + return; + } + + if (this._throwHttpErrors && !isHTTPError) { + this.destroy(error); + } else { + this.emit('error', error); + } + } +} diff --git a/source/as-promise/create-rejection.ts b/source/as-promise/create-rejection.ts new file mode 100644 index 000000000..9bf332c8b --- /dev/null +++ b/source/as-promise/create-rejection.ts @@ -0,0 +1,13 @@ +import {CancelableRequest} from './types'; + +export default function createRejection(error: Error): CancelableRequest { + const promise = Promise.reject(error) as CancelableRequest; + const returnPromise = (): CancelableRequest => promise; + + promise.json = returnPromise; + promise.text = returnPromise; + promise.buffer = returnPromise; + promise.on = returnPromise; + + return promise; +} diff --git a/source/as-promise/index.ts b/source/as-promise/index.ts new file mode 100644 index 000000000..f4ac38649 --- /dev/null +++ b/source/as-promise/index.ts @@ -0,0 +1,242 @@ +import {EventEmitter} from 'events'; +import getStream = require('get-stream'); +import PCancelable = require('p-cancelable'); +import calculateRetryDelay from './calculate-retry-delay'; +import { + NormalizedOptions, + CancelableRequest, + Response, + RequestError, + HTTPError, + ReadError, + ParseError +} from './types'; +import PromisableRequest, {parseBody} from './core'; +import proxyEvents from '../utils/proxy-events'; + +export default function asPromise(options: NormalizedOptions): CancelableRequest { + let retryCount = 0; + let body: Buffer; + const emitter = new EventEmitter(); + + const promise = new PCancelable((resolve, reject, onCancel) => { + const makeRequest = (): void => { + if (options.responseType === 'json' && options.headers.accept === undefined) { + options.headers.accept = 'application/json'; + } + + // Support retries + const {throwHttpErrors} = options; + if (!throwHttpErrors) { + options.throwHttpErrors = true; + } + + const request = new PromisableRequest(options.url, options); + request._throwHttpErrors = throwHttpErrors; + request._noPipe = true; + onCancel(() => request.destroy()); + + request.once('response', async (response: Response) => { + response.retryCount = retryCount; + + if (response.request.aborted) { + // Canceled while downloading - will throw a `CancelError` or `TimeoutError` error + return; + } + + const isOk = (): boolean => { + const {statusCode} = response; + const limitStatusCode = options.followRedirect ? 299 : 399; + + return (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304; + }; + + // Download body + try { + body = await getStream.buffer(request); + } catch (error) { + request._beforeError(new ReadError(error, options, response)); + return; + } + + // Parse body + try { + response.body = parseBody(body, options.responseType, options.encoding); + } catch (error) { + // Fallback to `utf8` + response.body = body.toString('utf8'); + + if (isOk()) { + const parseError = new ParseError(error, response, options); + request._beforeError(parseError); + return; + } + } + + try { + for (const [index, hook] of options.hooks.afterResponse.entries()) { + // @ts-ignore TS doesn't notice that CancelableRequest is a Promise + // eslint-disable-next-line no-await-in-loop + response = await hook(response, async (updatedOptions): CancelableRequest => { + request.destroy(); + + const typedOptions = PromisableRequest.normalizeArguments(undefined, { + ...updatedOptions, + retry: { + calculateDelay: () => 0 + }, + throwHttpErrors: false, + resolveBodyOnly: false + }, options); + + // Remove any further hooks for that request, because we'll call them anyway. + // The loop continues. We don't want duplicates (asPromise recursion). + typedOptions.hooks.afterResponse = typedOptions.hooks.afterResponse.slice(0, index); + + for (const hook of typedOptions.hooks.beforeRetry) { + // eslint-disable-next-line no-await-in-loop + await hook(typedOptions); + } + + const promise: CancelableRequest = asPromise(typedOptions); + + onCancel(() => { + promise.catch(() => {}); + promise.cancel(); + }); + + return promise; + }); + } + } catch (error) { + request._beforeError(error); + return; + } + + if (throwHttpErrors && !isOk()) { + reject(new HTTPError(response, options)); + return; + } + + resolve(options.resolveBodyOnly ? response.body as T : response as unknown as T); + }); + + request.once('error', (error: RequestError) => { + if (promise.isCanceled) { + return; + } + + if (!request.options) { + reject(error); + return; + } + + let backoff: number; + + retryCount++; + + try { + backoff = options.retry.calculateDelay({ + attemptCount: retryCount, + retryOptions: options.retry, + error, + computedValue: calculateRetryDelay({ + attemptCount: retryCount, + retryOptions: options.retry, + error, + computedValue: 0 + }) + }); + } catch (error_) { + // Don't emit the `response` event + request.destroy(); + + reject(new RequestError(error_.message, error, request.options)); + return; + } + + if (backoff) { + // Don't emit the `response` event + request.destroy(); + + const retry = async (): Promise => { + options.throwHttpErrors = throwHttpErrors; + + try { + for (const hook of options.hooks.beforeRetry) { + // eslint-disable-next-line no-await-in-loop + await hook(options, error, retryCount); + } + } catch (error_) { + // Don't emit the `response` event + request.destroy(); + + reject(new RequestError(error_.message, error, request.options)); + return; + } + + makeRequest(); + }; + + setTimeout(retry, backoff); + return; + } + + // The retry has not been made + retryCount--; + + if (error instanceof HTTPError) { + // It will be handled by the `response` event + return; + } + + // Don't emit the `response` event + request.destroy(); + + reject(error); + }); + + proxyEvents(request, emitter, [ + 'request', + 'response', + 'redirect', + 'uploadProgress', + 'downloadProgress' + ]); + }; + + makeRequest(); + }) as CancelableRequest; + + promise.on = (event: string, fn: (...args: any[]) => void) => { + emitter.on(event, fn); + return promise; + }; + + const shortcut = (responseType: NormalizedOptions['responseType']): CancelableRequest => { + const newPromise = (async () => { + await promise; + return parseBody(body, responseType); + })(); + + Object.defineProperties(newPromise, Object.getOwnPropertyDescriptors(promise)); + + return newPromise as CancelableRequest; + }; + + promise.json = () => { + if (body === undefined && options.headers.accept === undefined) { + options.headers.accept = 'application/json'; + } + + return shortcut('json'); + }; + + promise.buffer = () => shortcut('buffer'); + promise.text = () => shortcut('text'); + + return promise; +} + +export * from './types'; +export {PromisableRequest}; diff --git a/source/as-promise/types.ts b/source/as-promise/types.ts new file mode 100644 index 000000000..22893818b --- /dev/null +++ b/source/as-promise/types.ts @@ -0,0 +1,149 @@ +import PCancelable = require('p-cancelable'); +import {CancelError} from 'p-cancelable'; +import { + // Interfaces to be extended + Options as RequestOptions, + NormalizedOptions as RequestNormalizedOptions, + Defaults as RequestDefaults, + Hooks as RequestHooks, + Response as RequestResponse, + + // Errors to be exported + RequestError, + MaxRedirectsError, + CacheError, + UploadError, + TimeoutError, + HTTPError, + ReadError, + UnsupportedProtocolError, + + // Hooks to be exported + HookEvent as RequestHookEvent, + InitHook, + BeforeRequestHook, + BeforeRedirectHook, + BeforeErrorHook, + + // Other types to be exported + Progress, + Headers, + RequestFunction, + + // Types that will not be exported + Method, + RequestEvents +} from '../core'; + +export type ResponseType = 'json' | 'buffer' | 'text'; + +export type Response = RequestResponse; + +export interface RetryObject { + attemptCount: number; + retryOptions: RequiredRetryOptions; + error: TimeoutError | RequestError; + computedValue: number; +} + +export type RetryFunction = (retryObject: RetryObject) => number; + +export interface RequiredRetryOptions { + limit: number; + methods: Method[]; + statusCodes: number[]; + errorCodes: string[]; + calculateDelay: RetryFunction; + maxRetryAfter?: number; +} + +export type BeforeRetryHook = (options: NormalizedOptions, error?: RequestError, retryCount?: number) => void | Promise; +export type AfterResponseHook = (response: Response, retryWithMergedOptions: (options: Options) => CancelableRequest) => Response | CancelableRequest | Promise>; + +export interface Hooks extends RequestHooks { + beforeRetry?: BeforeRetryHook[]; + afterResponse?: AfterResponseHook[]; +} + +export interface PaginationOptions { + _pagination?: { + transform?: (response: Response) => Promise | T[]; + filter?: (item: T, allItems: T[]) => boolean; + paginate?: (response: Response) => Options | false; + shouldContinue?: (item: T, allItems: T[]) => boolean; + countLimit?: number; + }; +} + +export interface Options extends RequestOptions, PaginationOptions { + hooks?: Hooks; + responseType?: ResponseType; + resolveBodyOnly?: boolean; + retry?: Partial | number; + isStream?: boolean; +} + +export interface NormalizedOptions extends RequestNormalizedOptions { + hooks: Required; + responseType: ResponseType; + resolveBodyOnly: boolean; + retry: RequiredRetryOptions; + isStream: boolean; + _pagination?: Required['_pagination']>; +} + +export interface Defaults extends RequestDefaults { + hooks: Required; + responseType: ResponseType; + resolveBodyOnly: boolean; + retry: RequiredRetryOptions; + isStream: boolean; + _pagination?: Required['_pagination']>; +} + +export class ParseError extends RequestError { + declare readonly response: Response; + + constructor(error: Error, response: Response, options: NormalizedOptions) { + super(`${error.message} in "${options.url.toString()}"`, error, options); + this.name = 'ParseError'; + + Object.defineProperty(this, 'response', { + enumerable: false, + value: response + }); + } +} + +export interface CancelableRequest extends PCancelable, RequestEvents> { + json(): CancelableRequest; + buffer(): CancelableRequest; + text(): CancelableRequest; +} + +export type HookEvent = RequestHookEvent | 'beforeRetry' | 'afterResponse'; + +export { + RequestError, + MaxRedirectsError, + CacheError, + UploadError, + TimeoutError, + HTTPError, + ReadError, + UnsupportedProtocolError, + CancelError +}; + +export { + InitHook, + BeforeRequestHook, + BeforeRedirectHook, + BeforeErrorHook +}; + +export { + Progress, + Headers, + RequestFunction +}; diff --git a/source/as-stream.ts b/source/as-stream.ts deleted file mode 100644 index 4a942abda..000000000 --- a/source/as-stream.ts +++ /dev/null @@ -1,144 +0,0 @@ -import duplexer3 = require('duplexer3'); -import {IncomingMessage, ServerResponse} from 'http'; -import {Duplex as DuplexStream, PassThrough as PassThroughStream} from 'stream'; -import {HTTPError, ReadError} from './errors'; -import requestAsEventEmitter, {proxyEvents} from './request-as-event-emitter'; -import {GeneralError, GotEvents, NormalizedOptions, Response} from './types'; - -export class ProxyStream extends DuplexStream implements GotEvents> { - isFromCache?: boolean; -} - -export default function asStream(options: NormalizedOptions): ProxyStream { - const input = new PassThroughStream(); - const output = new PassThroughStream(); - const proxy = duplexer3(input, output) as ProxyStream; - const piped = new Set(); - let isFinished = false; - - options.retry.calculateDelay = () => 0; - - if (options.body || options.json || options.form) { - proxy.write = () => { - proxy.destroy(); - throw new Error('Got\'s stream is not writable when the `body`, `json` or `form` option is used'); - }; - } else if (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH' || (options.allowGetBody && options.method === 'GET')) { - options.body = input; - } else { - proxy.write = () => { - proxy.destroy(); - throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); - }; - } - - const emitter = requestAsEventEmitter(options); - - const emitError = async (error: GeneralError): Promise => { - try { - for (const hook of options.hooks.beforeError) { - // eslint-disable-next-line no-await-in-loop - error = await hook(error); - } - - proxy.emit('error', error); - } catch (error_) { - proxy.emit('error', error_); - } - }; - - // Cancels the request - proxy._destroy = (error, callback) => { - callback(error); - emitter.abort(); - }; - - emitter.on('response', (response: Response) => { - const {statusCode, isFromCache} = response; - proxy.isFromCache = isFromCache; - - if (options.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > 299)) { - emitError(new HTTPError(response, options)); - return; - } - - { - const read = proxy._read; - proxy._read = (...args) => { - isFinished = true; - - proxy._read = read; - return read.apply(proxy, args); - }; - } - - if (options.encoding) { - proxy.setEncoding(options.encoding); - } - - // We cannot use `stream.pipeline(...)` here, - // because if we did then `output` would throw - // the original error before throwing `ReadError`. - response.pipe(output); - response.once('error', error => { - emitError(new ReadError(error, options)); - }); - - for (const destination of piped) { - if (destination.headersSent) { - continue; - } - - for (const [key, value] of Object.entries(response.headers)) { - // Got gives *decompressed* data. Overriding `content-encoding` header would result in an error. - // It's not possible to decompress already decompressed data, is it? - const isAllowed = options.decompress ? key !== 'content-encoding' : true; - if (isAllowed) { - destination.setHeader(key, value!); - } - } - - destination.statusCode = response.statusCode; - } - - proxy.emit('response', response); - }); - - proxyEvents(proxy, emitter); - emitter.on('error', (error: GeneralError) => proxy.emit('error', error)); - - const pipe = proxy.pipe.bind(proxy); - const unpipe = proxy.unpipe.bind(proxy); - - proxy.pipe = (destination, options) => { - if (isFinished) { - throw new Error('Failed to pipe. The response has been emitted already.'); - } - - pipe(destination, options); - - if (destination instanceof ServerResponse) { - piped.add(destination); - } - - return destination; - }; - - proxy.unpipe = stream => { - piped.delete(stream as ServerResponse); - return unpipe(stream); - }; - - proxy.on('pipe', source => { - if (source instanceof IncomingMessage) { - options.headers = { - ...source.headers, - ...options.headers - }; - } - }); - - proxy.isFromCache = undefined; - - return proxy; -} diff --git a/source/core/index.ts b/source/core/index.ts new file mode 100644 index 000000000..1d606a327 --- /dev/null +++ b/source/core/index.ts @@ -0,0 +1,1462 @@ +import {promisify} from 'util'; +import {Duplex, Writable, Readable} from 'stream'; +import {ReadStream} from 'fs'; +import {URL, URLSearchParams} from 'url'; +import {Socket} from 'net'; +import {SecureContextOptions} from 'tls'; +import http = require('http'); +import {ClientRequest, RequestOptions, IncomingMessage, ServerResponse, request as httpRequest} from 'http'; +import https = require('https'); +import timer, {ClientRequestWithTimings, Timings} from '@szmarczak/http-timer'; +import decompressResponse = require('decompress-response'); +import CacheableLookup from 'cacheable-lookup'; +import CacheableRequest = require('cacheable-request'); +// @ts-ignore Missing types +import http2wrapper = require('http2-wrapper'); +import lowercaseKeys = require('lowercase-keys'); +import ResponseLike = require('responselike'); +import getStream = require('get-stream'); +import is, {assert} from '@sindresorhus/is'; +import getBodySize from '../utils/get-body-size'; +import isFormData from '../utils/is-form-data'; +import proxyEvents from '../utils/proxy-events'; +import timedOut, {Delays, TimeoutError as TimedOutTimeoutError} from '../utils/timed-out'; +import urlToOptions from '../utils/url-to-options'; + +type HttpRequestFunction = typeof httpRequest; +type Error = NodeJS.ErrnoException; + +const kRequest = Symbol('request'); +const kResponse = Symbol('response'); +const kResponseSize = Symbol('responseSize'); +const kDownloadedSize = Symbol('downloadedSize'); +const kBodySize = Symbol('bodySize'); +const kUploadedSize = Symbol('uploadedSize'); +const kServerResponsesPiped = Symbol('serverResponsesPiped'); +const kUnproxyEvents = Symbol('unproxyEvents'); +const kIsFromCache = Symbol('isFromCache'); +const kCancelTimeouts = Symbol('cancelTimeouts'); +const kStartedReading = Symbol('startedReading'); +export const kIsNormalizedAlready = Symbol('isNormalizedAlready'); + +const supportsBrotli = is.string((process.versions as any).brotli); + +export interface Agents { + http?: http.Agent; + https?: https.Agent; + http2?: unknown; +} + +export const withoutBody: ReadonlySet = new Set(['GET', 'HEAD']); + +export interface ToughCookieJar { + getCookieString(currentUrl: string, options: {[key: string]: unknown}, cb: (err: Error | null, cookies: string) => void): void; + getCookieString(url: string, callback: (error: Error | null, cookieHeader: string) => void): void; + setCookie(cookieOrString: unknown, currentUrl: string, options: {[key: string]: unknown}, cb: (err: Error | null, cookie: unknown) => void): void; + setCookie(rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void): void; +} + +export interface PromiseCookieJar { + getCookieString(url: string): Promise; + setCookie(rawCookie: string, url: string): Promise; +} + +export type Method = + | 'GET' + | 'POST' + | 'PUT' + | 'PATCH' + | 'HEAD' + | 'DELETE' + | 'OPTIONS' + | 'TRACE' + | 'get' + | 'post' + | 'put' + | 'patch' + | 'head' + | 'delete' + | 'options' + | 'trace'; + +type Promisable = T | Promise; + +export type InitHook = (options: Options & {url: string | URL}) => Promisable; +export type BeforeRequestHook = (options: NormalizedOptions) => Promisable; +export type BeforeRedirectHook = (options: NormalizedOptions, response: Response) => Promisable; +export type BeforeErrorHook = (error: RequestError) => Promisable; + +export interface Hooks { + init?: InitHook[]; + beforeRequest?: BeforeRequestHook[]; + beforeRedirect?: BeforeRedirectHook[]; + beforeError?: BeforeErrorHook[]; +} + +export type HookEvent = 'init' | 'beforeRequest' | 'beforeRedirect' | 'beforeError'; + +export const knownHookEvents: HookEvent[] = ['init', 'beforeRequest', 'beforeRedirect', 'beforeError']; + +type AcceptableResponse = IncomingMessage | ResponseLike; +type AcceptableRequestResult = AcceptableResponse | ClientRequest | Promise | undefined; + +export type RequestFunction = (url: URL, options: RequestOptions, callback?: (response: AcceptableResponse) => void) => AcceptableRequestResult; + +export type Headers = Record; + +export interface Options extends SecureContextOptions { + request?: RequestFunction; + agent?: Agents | false; + decompress?: boolean; + timeout?: Delays | number; + prefixUrl?: string | URL; + body?: unknown; + form?: {[key: string]: any}; + json?: {[key: string]: any}; + url?: string | URL; + cookieJar?: PromiseCookieJar | ToughCookieJar; + ignoreInvalidCookies?: boolean; + encoding?: BufferEncoding; + searchParams?: string | {[key: string]: string | number | boolean | null} | URLSearchParams; + dnsCache?: CacheableLookup | boolean; + context?: object; + hooks?: Hooks; + followRedirect?: boolean; + maxRedirects?: number; + cache?: string | CacheableRequest.StorageAdapter; + throwHttpErrors?: boolean; + username?: string; + password?: string; + http2?: boolean; + allowGetBody?: boolean; + lookup?: CacheableLookup['lookup']; + rejectUnauthorized?: boolean; + headers?: Headers; + methodRewriting?: boolean; + + // From http.RequestOptions + localAddress?: string; + socketPath?: string; + method?: string; + createConnection?: (options: http.RequestOptions, oncreate: (error: Error, socket: Socket) => void) => Socket; +} + +export interface NormalizedOptions extends Options { + method: Method; + url: URL; + timeout: Delays; + prefixUrl: string; + ignoreInvalidCookies: boolean; + decompress: boolean; + searchParams?: URLSearchParams; + cookieJar?: PromiseCookieJar; + headers: Headers; + context: object; + hooks: Required; + followRedirect: boolean; + maxRedirects: number; + throwHttpErrors: boolean; + dnsCache?: CacheableLookup; + cacheableRequest?: (options: string | URL | http.RequestOptions, callback?: (response: http.ServerResponse | ResponseLike) => void) => CacheableRequest.Emitter; + http2: boolean; + allowGetBody: boolean; + rejectUnauthorized: boolean; + lookup?: CacheableLookup['lookup']; + methodRewriting: boolean; + username: string; + password: string; + [kRequest]: HttpRequestFunction; + [kIsNormalizedAlready]?: boolean; +} + +export interface Defaults { + timeout: Delays; + prefixUrl: string; + method: Method; + ignoreInvalidCookies: boolean; + decompress: boolean; + context: object; + cookieJar?: PromiseCookieJar | ToughCookieJar; + dnsCache?: CacheableLookup; + headers: Headers; + hooks: Required; + followRedirect: boolean; + maxRedirects: number; + cache?: string | CacheableRequest.StorageAdapter; + throwHttpErrors: boolean; + http2: boolean; + allowGetBody: boolean; + rejectUnauthorized: boolean; + methodRewriting: boolean; + + // Optional + agent?: Agents | false; + request?: RequestFunction; + searchParams?: URLSearchParams; + lookup?: CacheableLookup['lookup']; + localAddress?: string; + createConnection?: Options['createConnection']; +} + +export interface Progress { + percent: number; + transferred: number; + total?: number; +} + +export interface PlainResponse extends IncomingMessage { + requestUrl: string; + redirectUrls: string[]; + request: Request; + ip?: string; + isFromCache: boolean; + statusCode: number; + url: string; +} + +// For Promise support +export interface Response extends PlainResponse { + body: T; + retryCount: number; +} + +export interface RequestEvents { + on(name: 'request', listener: (request: http.ClientRequest) => void): T; + on(name: 'response', listener: (response: Response) => void): T; + on(name: 'redirect', listener: (response: Response, nextOptions: NormalizedOptions) => void): T; + on(name: 'uploadProgress' | 'downloadProgress', listener: (progress: Progress) => void): T; +} + +function validateSearchParams(searchParams: Record): asserts searchParams is Record { + // eslint-disable-next-line guard-for-in + for (const key in searchParams) { + const value = searchParams[key]; + + if (!is.string(value) && !is.number(value) && !is.boolean(value) && !is.null_(value)) { + throw new TypeError(`The \`searchParams\` value '${String(value)}' must be a string, number, boolean or null`); + } + } +} + +function isClientRequest(clientRequest: unknown): clientRequest is ClientRequest { + return is.object(clientRequest) && !('statusCode' in clientRequest); +} + +const cacheFn = async (url: URL, options: RequestOptions): Promise => new Promise((resolve, reject) => { + // TODO: Remove `utils/url-to-options.ts` when `cacheable-request` is fixed + Object.assign(options, urlToOptions(url)); + + // `http-cache-semantics` checks this + delete (options as unknown as NormalizedOptions).url; + + // TODO: `cacheable-request` is incorrectly typed + const cacheRequest = (options as Pick).cacheableRequest!(options, resolve as any); + + // Restore options + (options as unknown as NormalizedOptions).url = url; + + cacheRequest.once('error', (error: Error) => { + if (error instanceof CacheableRequest.RequestError) { + // TODO: `options` should be `normalizedOptions` + reject(new RequestError(error.message, error, options as unknown as NormalizedOptions)); + return; + } + + // TODO: `options` should be `normalizedOptions` + reject(new CacheError(error, options as unknown as NormalizedOptions)); + }); + cacheRequest.once('request', resolve); +}); + +const waitForOpenFile = async (file: ReadStream): Promise => new Promise((resolve, reject) => { + const onError = (error: Error): void => { + reject(error); + }; + + file.once('error', onError); + file.once('open', () => { + file.off('error', onError); + resolve(); + }); +}); + +const redirectCodes: ReadonlySet = new Set([300, 301, 302, 303, 304, 307, 308]); + +type NonEnumerableProperty = 'context' | 'body' | 'json' | 'form'; +const nonEnumerableProperties: NonEnumerableProperty[] = [ + 'context', + 'body', + 'json', + 'form' +]; + +const setNonEnumerableProperties = (sources: Array, to: Options): void => { + // Non enumerable properties shall not be merged + const properties: Partial<{[Key in NonEnumerableProperty]: any}> = {}; + + for (const source of sources) { + if (!source) { + continue; + } + + for (const name of nonEnumerableProperties) { + if (!(name in source)) { + continue; + } + + properties[name] = { + writable: true, + configurable: true, + enumerable: false, + // @ts-ignore TS doesn't see the check above + value: source[name] + }; + } + } + + Object.defineProperties(to, properties); +}; + +export class RequestError extends Error { + code?: string; + stack!: string; + declare readonly options: NormalizedOptions; + readonly response?: Response; + readonly request?: Request; + readonly timings?: Timings; + + constructor(message: string, error: Partial, options: NormalizedOptions, requestOrResponse?: Request | Response) { + super(message); + Error.captureStackTrace(this, this.constructor); + + this.name = 'RequestError'; + this.code = error.code; + + Object.defineProperty(this, 'options', { + // This fails because of TS 3.7.2 useDefineForClassFields + // Ref: https://github.com/microsoft/TypeScript/issues/34972 + enumerable: false, + value: options + }); + + if (requestOrResponse instanceof IncomingMessage) { + Object.defineProperty(this, 'response', { + enumerable: false, + value: requestOrResponse + }); + + requestOrResponse = requestOrResponse.request; + } + + if (requestOrResponse instanceof Request) { + Object.defineProperty(this, 'request', { + enumerable: false, + value: requestOrResponse + }); + + this.timings = requestOrResponse.timings; + } + + // Recover the original stacktrace + if (!is.undefined(error.stack)) { + const indexOfMessage = this.stack.indexOf(this.message) + this.message.length; + const thisStackTrace = this.stack.slice(indexOfMessage).split('\n').reverse(); + const errorStackTrace = error.stack.slice(error.stack.indexOf(error.message!) + error.message!.length).split('\n').reverse(); + + // Remove duplicated traces + while (errorStackTrace.length !== 0 && errorStackTrace[0] === thisStackTrace[0]) { + thisStackTrace.shift(); + } + + this.stack = `${this.stack.slice(0, indexOfMessage)}${thisStackTrace.reverse().join('\n')}${errorStackTrace.reverse().join('\n')}`; + } + } +} + +export class MaxRedirectsError extends RequestError { + declare readonly response: Response; + + constructor(response: Response, maxRedirects: number, options: NormalizedOptions) { + super(`Redirected ${maxRedirects} times. Aborting.`, {}, options); + this.name = 'MaxRedirectsError'; + + Object.defineProperty(this, 'response', { + enumerable: false, + value: response + }); + } +} + +export class HTTPError extends RequestError { + declare readonly response: Response; + + constructor(response: Response, options: NormalizedOptions) { + super(`Response code ${response.statusCode} (${response.statusMessage!})`, {}, options); + this.name = 'HTTPError'; + + Object.defineProperty(this, 'response', { + enumerable: false, + value: response + }); + } +} + +export class CacheError extends RequestError { + constructor(error: Error, options: NormalizedOptions) { + super(error.message, error, options); + this.name = 'CacheError'; + } +} + +export class UploadError extends RequestError { + constructor(error: Error, options: NormalizedOptions, request: Request) { + super(error.message, error, options, request); + this.name = 'UploadError'; + } +} + +export class TimeoutError extends RequestError { + readonly timings: Timings; + readonly event: string; + + constructor(error: TimedOutTimeoutError, timings: Timings, options: NormalizedOptions) { + super(error.message, error, options); + this.name = 'TimeoutError'; + this.event = error.event; + this.timings = timings; + } +} + +export class ReadError extends RequestError { + constructor(error: Error, options: NormalizedOptions, response: Response) { + super(error.message, error, options, response); + this.name = 'ReadError'; + } +} + +export class UnsupportedProtocolError extends RequestError { + constructor(options: NormalizedOptions) { + super(`Unsupported protocol "${options.url.protocol}"`, {}, options); + this.name = 'UnsupportedProtocolError'; + } +} + +export default class Request extends Duplex implements RequestEvents { + ['constructor']: typeof Request; + + declare [kUnproxyEvents]: () => void; + declare _cannotHaveBody: boolean; + [kDownloadedSize]: number; + [kUploadedSize]: number; + [kBodySize]?: number; + [kServerResponsesPiped]: Set; + [kIsFromCache]?: boolean; + [kStartedReading]?: boolean; + [kCancelTimeouts]?: () => void; + [kResponseSize]?: number; + [kResponse]?: IncomingMessage; + [kRequest]?: ClientRequest; + _noPipe?: boolean; + + declare options: NormalizedOptions; + declare requestUrl: string; + finalized: boolean; + redirects: string[]; + + constructor(url: string | URL, options: Options = {}, defaults?: Defaults) { + super({ + // It needs to be zero because we're just proxying the data to another stream + highWaterMark: 0 + }); + + this[kDownloadedSize] = 0; + this[kUploadedSize] = 0; + this.finalized = false; + this[kServerResponsesPiped] = new Set(); + this.redirects = []; + + const unlockWrite = (): void => this._unlockWrite(); + const lockWrite = (): void => this._lockWrite(); + + this.on('pipe', (source: Writable) => { + source.prependListener('data', unlockWrite); + source.on('data', lockWrite); + + source.prependListener('end', unlockWrite); + source.on('end', lockWrite); + }); + + this.on('unpipe', (source: Writable) => { + source.off('data', unlockWrite); + source.off('data', lockWrite); + + source.off('end', unlockWrite); + source.off('end', lockWrite); + }); + + this.on('pipe', source => { + if (source instanceof IncomingMessage) { + this.options.headers = { + ...source.headers, + ...this.options.headers + }; + } + }); + + (async (nonNormalizedOptions: Options) => { + try { + { + const {json, body, form} = nonNormalizedOptions; + if (json || body || form) { + this._lockWrite(); + } + } + + if (nonNormalizedOptions.body instanceof ReadStream) { + await waitForOpenFile(nonNormalizedOptions.body); + } + + const initHooks = nonNormalizedOptions.hooks?.init; + const hasInitHooks = initHooks && initHooks.length !== 0; + if (hasInitHooks) { + nonNormalizedOptions.url = url; + + for (const hook of initHooks!) { + // eslint-disable-next-line no-await-in-loop + await hook(nonNormalizedOptions as Options & {url: string | URL}); + } + + url = nonNormalizedOptions.url; + nonNormalizedOptions.url = undefined; + } + + if (kIsNormalizedAlready in nonNormalizedOptions && !hasInitHooks) { + this.options = nonNormalizedOptions as NormalizedOptions; + } else { + // @ts-ignore Common TypeScript bug saying that `this.constructor` is not accessible + this.options = this.constructor.normalizeArguments(url, nonNormalizedOptions, defaults); + } + + const {options} = this; + + if (!options.url) { + throw new TypeError('Missing `url` property'); + } + + this.requestUrl = options.url.toString(); + decodeURI(this.requestUrl); + + await this._finalizeBody(); + await this._makeRequest(); + + this.finalized = true; + this.emit('finalized'); + } catch (error) { + if (error instanceof RequestError) { + this._beforeError(error); + return; + } + + this.destroy(error); + } + })(options); + } + + static normalizeArguments(url?: string | URL, options?: Options, defaults?: Defaults): NormalizedOptions { + const rawOptions = options; + + if (is.object(url) && !is.urlInstance(url)) { + options = {...defaults as NormalizedOptions, ...(url as Options), ...options}; + } else { + if (url && options && options.url) { + throw new TypeError('The `url` option is mutually exclusive with the `input` argument'); + } + + options = {...defaults as NormalizedOptions, ...options}; + + if (url) { + options.url = url; + } + } + + if (rawOptions && defaults) { + for (const key in rawOptions) { + // @ts-ignore Dear TypeScript, all object keys are strings (or symbols which are NOT enumerable). + if (is.undefined(rawOptions[key]) && !is.undefined(defaults[key])) { + // @ts-ignore See the note above + options[key] = defaults[key]; + } + } + } + + // Disallow `options.path` and `options.pathname` + if ( + 'path' in options || + 'pathname' in options || + 'hostname' in options || + 'host' in options || + 'port' in options || + 'search' in options || + 'protocol' in options || + 'auth' in options + ) { + throw new TypeError('The legacy `url.Url` has been deprecated. Use `URL` instead.'); + } + + // Verify types + if (is.null_(options.encoding)) { + throw new TypeError('To get a Buffer, set `options.responseType` to `buffer` instead'); + } + + assert.any([is.string, is.undefined], options.method); + assert.any([is.object, is.undefined], options.headers); + assert.any([is.string, is.urlInstance, is.undefined], options.prefixUrl); + assert.any([is.object, is.undefined], options.cookieJar); + assert.any([is.object, is.string, is.undefined], options.searchParams); + assert.any([is.object, is.string, is.undefined], options.cache); + assert.any([is.object, is.number, is.undefined], options.timeout); + assert.any([is.object, is.undefined], options.context); + assert.any([is.object, is.undefined], options.hooks); + assert.any([is.boolean, is.undefined], options.decompress); + assert.any([is.boolean, is.undefined], options.ignoreInvalidCookies); + assert.any([is.string, is.undefined], options.encoding); + assert.any([is.boolean, is.undefined], options.followRedirect); + assert.any([is.number, is.undefined], options.maxRedirects); + assert.any([is.boolean, is.undefined], options.throwHttpErrors); + assert.any([is.boolean, is.undefined], options.http2); + assert.any([is.boolean, is.undefined], options.allowGetBody); + assert.any([is.boolean, is.undefined], options.rejectUnauthorized); + + // `options.method` + if (is.string(options.method)) { + options.method = options.method.toUpperCase(); + } else { + options.method = 'GET'; + } + + // `options.headers` + if (is.undefined(options.headers)) { + options.headers = {}; + } else { + options.headers = lowercaseKeys({...(defaults?.headers), ...options.headers}); + } + + // `options.prefixUrl` & `options.url` + if (options.prefixUrl) { + options.prefixUrl = options.prefixUrl.toString(); + + if (options.prefixUrl !== '' && !options.prefixUrl.endsWith('/')) { + options.prefixUrl += '/'; + } + } else { + options.prefixUrl = ''; + } + + if (is.string(options.url)) { + if (options.url.startsWith('/')) { + throw new Error('`input` must not start with a slash when using `prefixUrl`'); + } + + options.url = new URL(options.prefixUrl + options.url); + } else if (is.undefined(options.url) && options.prefixUrl !== '') { + options.url = new URL(options.prefixUrl); + } + + if (options.url) { + // Make it possible to change `options.prefixUrl` + let {prefixUrl} = options; + Object.defineProperty(options, 'prefixUrl', { + set: (value: string) => { + const url = options!.url as URL; + + if (!url.href.startsWith(value)) { + throw new Error(`Cannot change \`prefixUrl\` from ${prefixUrl} to ${value}: ${url.href}`); + } + + options!.url = new URL(value + url.href.slice(prefixUrl.length)); + prefixUrl = value; + }, + get: () => prefixUrl + }); + + // Protocol check + let {protocol} = options.url; + + if (protocol === 'unix:') { + protocol = 'http:'; + + options.url = new URL(`http://unix${options.url.pathname}${options.url.search}`); + } + + if (protocol !== 'http:' && protocol !== 'https:') { + throw new UnsupportedProtocolError(options as NormalizedOptions); + } + } + + // `options.username` & `options.password` + options.username = options.username ?? ''; + options.password = options.password ?? ''; + + if (options.url) { + options.url.username = options.username; + options.url.password = options.password; + } + + // `options.cookieJar` + if (options.cookieJar) { + let {setCookie, getCookieString} = options.cookieJar; + + // Horrible `tough-cookie` check + if (setCookie.length === 4 && getCookieString.length === 0) { + setCookie = promisify(setCookie.bind(options.cookieJar)); + getCookieString = promisify(getCookieString.bind(options.cookieJar)); + } else if (setCookie.length !== 2) { + throw new TypeError('`options.cookieJar.setCookie` needs to be an async function with 2 arguments'); + } else if (getCookieString.length !== 1) { + throw new TypeError('`options.cookieJar.getCookieString` needs to be an async function with 1 argument'); + } + + options.cookieJar = {setCookie, getCookieString}; + } + + // `options.searchParams` + if (options.searchParams) { + if (!is.string(options.searchParams) && !(options.searchParams instanceof URLSearchParams)) { + validateSearchParams(options.searchParams); + } + + options.searchParams = new URLSearchParams(options.searchParams as Record); + + // `normalizeArguments()` is also used to merge options + const defaultsAsOptions = defaults as Options | undefined; + if (defaultsAsOptions && defaultsAsOptions.searchParams instanceof URLSearchParams) { + defaultsAsOptions.searchParams.forEach((value, key) => { + (options!.searchParams as URLSearchParams).append(key, value); + }); + } + + if (options.url) { + options.url.search = options.searchParams.toString(); + } + } + + // `options.cache` + if (options.cache && !(options as NormalizedOptions).cacheableRequest) { + // Better memory management, so we don't have to generate a new object every time + (options as NormalizedOptions).cacheableRequest = new CacheableRequest( + ((requestOptions: RequestOptions, handler?: (response: IncomingMessage) => void): ClientRequest => (requestOptions as Pick)[kRequest](requestOptions, handler)) as HttpRequestFunction, + options.cache + ); + } + + // `options.dnsCache` + if (options.dnsCache === true) { + options.dnsCache = new CacheableLookup(); + } else if (!is.undefined(options.dnsCache) && options.dnsCache !== false && !(options.dnsCache instanceof CacheableLookup)) { + throw new TypeError(`Parameter \`dnsCache\` must be a CacheableLookup instance or a boolean, got ${is(options.dnsCache)}`); + } + + // `options.timeout` + if (is.number(options.timeout)) { + options.timeout = {request: options.timeout}; + } else { + options.timeout = {...options.timeout}; + } + + if (defaults) { + options.timeout = { + ...defaults.timeout, + ...options.timeout + }; + } + + // `options.context` + if (!options.context) { + options.context = {}; + } + + // `options.hooks` + options.hooks = {...options.hooks}; + + for (const event of knownHookEvents) { + if (event in options.hooks) { + if (Array.isArray(options.hooks[event])) { + // See https://github.com/microsoft/TypeScript/issues/31445#issuecomment-576929044 + (options.hooks as any)[event] = [...options.hooks[event]!]; + } else { + throw new TypeError(`Parameter \`${event}\` must be an Array, got ${is(options.hooks[event])}`); + } + } else { + options.hooks[event] = []; + } + } + + if (defaults) { + for (const event of knownHookEvents) { + // See https://github.com/microsoft/TypeScript/issues/31445#issuecomment-576929044 + (options.hooks as any)[event] = [ + ...defaults.hooks[event], + ...options.hooks[event]! + ]; + } + } + + // Other options + if ('followRedirects' in options) { + throw new TypeError('The `followRedirects` option does not exist. Use `followRedirect` instead.'); + } + + if (options.agent) { + for (const key in options.agent) { + if (key !== 'http' && key !== 'https' && key !== 'http2') { + throw new TypeError(`Expected the \`options.agent\` properties to be \`http\`, \`https\` or \`http2\`, got \`${key}\``); + } + } + } + + options.decompress = Boolean(options.decompress); + options.ignoreInvalidCookies = Boolean(options.ignoreInvalidCookies); + options.followRedirect = Boolean(options.followRedirect); + options.maxRedirects = options.maxRedirects ?? 0; + options.throwHttpErrors = Boolean(options.throwHttpErrors); + options.http2 = Boolean(options.http2); + options.allowGetBody = Boolean(options.allowGetBody); + options.rejectUnauthorized = Boolean(options.rejectUnauthorized); + + // Set non-enumerable properties + setNonEnumerableProperties([defaults, options], options); + + return options as NormalizedOptions; + } + + _lockWrite(): void { + const onLockedWrite = (): never => { + throw new TypeError('The payload has been already provided'); + }; + + this.write = onLockedWrite; + this.end = onLockedWrite; + } + + _unlockWrite(): void { + this.write = super.write; + this.end = super.end; + } + + async _finalizeBody(): Promise { + const {options} = this; + const {headers} = options; + + const isForm = !is.undefined(options.form); + const isJSON = !is.undefined(options.json); + const isBody = !is.undefined(options.body); + const hasPayload = isForm || isJSON || isBody; + const cannotHaveBody = withoutBody.has(options.method) && !(options.method === 'GET' && options.allowGetBody); + + this._cannotHaveBody = cannotHaveBody; + + if (hasPayload) { + if (cannotHaveBody) { + throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); + } + + if ([isBody, isForm, isJSON].filter(isTrue => isTrue).length > 1) { + throw new TypeError('The `body`, `json` and `form` options are mutually exclusive'); + } + + if ( + isBody && + !(options.body instanceof Readable) && + !is.string(options.body) && + !is.buffer(options.body) && + !isFormData(options.body) + ) { + throw new TypeError('The `body` option must be a stream.Readable, string or Buffer'); + } + + if (isForm && !is.object(options.form)) { + throw new TypeError('The `form` option must be an Object'); + } + + { + // Serialize body + const noContentType = !is.string(headers['content-type']); + + if (isBody) { + // Special case for https://github.com/form-data/form-data + if (isFormData(options.body) && noContentType) { + headers['content-type'] = `multipart/form-data; boundary=${options.body.getBoundary()}`; + } + } else if (isForm) { + if (noContentType) { + headers['content-type'] = 'application/x-www-form-urlencoded'; + } + + options.body = (new URLSearchParams(options.form as Record)).toString(); + } else { + if (noContentType) { + headers['content-type'] = 'application/json'; + } + + options.body = JSON.stringify(options.json); + } + + const uploadBodySize = await getBodySize(options); + + // See https://tools.ietf.org/html/rfc7230#section-3.3.2 + // A user agent SHOULD send a Content-Length in a request message when + // no Transfer-Encoding is sent and the request method defines a meaning + // for an enclosed payload body. For example, a Content-Length header + // field is normally sent in a POST request even when the value is 0 + // (indicating an empty payload body). A user agent SHOULD NOT send a + // Content-Length header field when the request message does not contain + // a payload body and the method semantics do not anticipate such a + // body. + if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding'])) { + if (!cannotHaveBody && !is.undefined(uploadBodySize)) { + headers['content-length'] = String(uploadBodySize); + } + } + } + } else if (cannotHaveBody) { + this._lockWrite(); + } else { + this._unlockWrite(); + } + + this[kBodySize] = Number(headers['content-length']) || undefined; + } + + async _onResponse(response: IncomingMessage): Promise { + const {options} = this; + const {url} = options; + + if (options.decompress) { + response = decompressResponse(response); + } + + const statusCode = response.statusCode!; + const typedResponse = response as Response; + + typedResponse.statusMessage = typedResponse.statusMessage === '' ? http.STATUS_CODES[statusCode] : typedResponse.statusMessage; + typedResponse.url = options.url.toString(); + typedResponse.requestUrl = this.requestUrl; + typedResponse.redirectUrls = this.redirects; + typedResponse.request = this; + typedResponse.isFromCache = (response as any).fromCache || false; + typedResponse.ip = this.ip; + + this[kIsFromCache] = typedResponse.isFromCache; + + const rawCookies = response.headers['set-cookie']; + if (is.object(options.cookieJar) && rawCookies) { + let promises: Array> = rawCookies.map(async (rawCookie: string) => (options.cookieJar as PromiseCookieJar).setCookie(rawCookie, url.toString())); + + if (options.ignoreInvalidCookies) { + promises = promises.map(async p => p.catch(() => {})); + } + + try { + await Promise.all(promises); + } catch (error) { + this._beforeError(error); + return; + } + } + + if (options.followRedirect && response.headers.location && redirectCodes.has(statusCode)) { + // We're being redirected, we don't care about the response. + // It'd be besto to abort the request, but we can't because + // we would have to sacrifice the TCP connection. We don't want that. + response.resume(); + + if (this[kRequest]) { + this[kCancelTimeouts]!(); + + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete this[kRequest]; + this[kUnproxyEvents](); + } + + const shouldBeGet = statusCode === 303 && options.method !== 'GET' && options.method !== 'HEAD'; + if (shouldBeGet || !options.methodRewriting) { + // Server responded with "see other", indicating that the resource exists at another location, + // and the client should request it from that location via GET or HEAD. + options.method = 'GET'; + + if ('body' in options) { + delete options.body; + } + + if ('json' in options) { + delete options.json; + } + + if ('form' in options) { + delete options.form; + } + } + + if (this.redirects.length >= options.maxRedirects) { + this._beforeError(new MaxRedirectsError(typedResponse, options.maxRedirects, options)); + return; + } + + try { + // Handles invalid URLs. See https://github.com/sindresorhus/got/issues/604 + const redirectBuffer = Buffer.from(response.headers.location, 'binary').toString(); + const redirectUrl = new URL(redirectBuffer, url); + const redirectString = redirectUrl.toString(); + decodeURI(redirectString); + + // Redirecting to a different site, clear cookies. + if (redirectUrl.hostname !== url.hostname && 'cookie' in options.headers) { + delete options.headers.cookie; + delete options.username; + delete options.password; + } + + this.redirects.push(redirectString); + options.url = redirectUrl; + + for (const hook of options.hooks.beforeRedirect) { + // eslint-disable-next-line no-await-in-loop + await hook(options, typedResponse); + } + + this.emit('redirect', typedResponse, options); + + await this._makeRequest(); + } catch (error) { + this._beforeError(error); + return; + } + + return; + } + + const limitStatusCode = options.followRedirect ? 299 : 399; + const isOk = (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304; + if (options.throwHttpErrors && !isOk) { + await this._beforeError(new HTTPError(typedResponse, options)); + + if (this.destroyed) { + return; + } + } + + this[kResponseSize] = Number(response.headers['content-length']) || undefined; + this[kResponse] = response; + + // We need to call `_read()` only when the Request stream is flowing + response.on('readable', () => { + if ((this as any).readableFlowing) { + this._read(); + } + }); + + this.on('resume', () => { + response.resume(); + }); + + this.on('pause', () => { + response.pause(); + }); + + response.once('end', () => { + this[kResponseSize] = this[kDownloadedSize]; + this.emit('downloadProgress', this.downloadProgress); + + this.push(null); + }); + + response.on('error', (error: Error) => { + this._beforeError(new ReadError(error, options, response as Response)); + }); + + for (const destination of this[kServerResponsesPiped]) { + if (destination.headersSent) { + continue; + } + + // eslint-disable-next-line guard-for-in + for (const key in response.headers) { + const isAllowed = options.decompress ? key !== 'content-encoding' : true; + const value = response.headers[key]; + + if (isAllowed) { + destination.setHeader(key, value!); + } + } + + destination.statusCode = statusCode; + } + + this.emit('downloadProgress', this.downloadProgress); + this.emit('response', response); + } + + _onRequest(request: ClientRequest): void { + const {options} = this; + const {timeout, url} = options; + + timer(request); + + this[kCancelTimeouts] = timedOut(request, timeout, url); + + request.once('response', response => { + this._onResponse(response); + }); + + request.once('error', (error: Error) => { + if (error instanceof TimedOutTimeoutError) { + error = new TimeoutError(error, this.timings!, options); + } else { + error = new RequestError(error.message, error, options, this); + } + + this._beforeError(error as RequestError); + }); + + this[kUnproxyEvents] = proxyEvents(request, this, [ + 'socket', + 'abort', + 'connect', + 'continue', + 'information', + 'upgrade', + 'timeout' + ]); + + this[kRequest] = request; + + this.emit('uploadProgress', this.uploadProgress); + + // Send body + const currentRequest = this.redirects.length === 0 ? this : request; + if (is.nodeStream(options.body)) { + options.body.pipe(currentRequest); + options.body.once('error', (error: NodeJS.ErrnoException) => { + this._beforeError(new UploadError(error, options, this)); + }); + + options.body.once('end', () => { + delete options.body; + }); + } else { + this._unlockWrite(); + + if (!is.undefined(options.body)) { + this._writeRequest(options.body, null as unknown as string, () => {}); + currentRequest.end(); + + this._lockWrite(); + } else if (this._cannotHaveBody || this._noPipe) { + currentRequest.end(); + + this._lockWrite(); + } + } + + this.emit('request', request); + } + + async _makeRequest(): Promise { + const {options} = this; + const {url, headers, request, agent, timeout} = options; + + for (const key in headers) { + if (is.undefined(headers[key])) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete headers[key]; + } else if (is.null_(headers[key])) { + throw new TypeError(`Use \`undefined\` instead of \`null\` to delete the \`${key}\` header`); + } + } + + if (options.decompress && is.undefined(headers['accept-encoding'])) { + headers['accept-encoding'] = supportsBrotli ? 'gzip, deflate, br' : 'gzip, deflate'; + } + + // Set cookies + if (options.cookieJar) { + const cookieString: string = await options.cookieJar.getCookieString(options.url.toString()); + + if (is.nonEmptyString(cookieString)) { + options.headers.cookie = cookieString; + } + } + + for (const hook of options.hooks.beforeRequest) { + // eslint-disable-next-line no-await-in-loop + const result = await hook(options); + + if (!is.undefined(result)) { + // @ts-ignore Skip the type mismatch to support abstract responses + options.request = () => result; + break; + } + } + + if (options.dnsCache && !('lookup' in options)) { + options.lookup = options.dnsCache.lookup; + } + + // UNIX sockets + if (url.hostname === 'unix') { + const matches = /(?.+?):(?.+)/.exec(`${url.pathname}${url.search}`); + + if (matches?.groups) { + const {socketPath, path} = matches.groups; + + Object.assign(options, { + socketPath, + path, + host: '' + }); + } + } + + const isHttps = url.protocol === 'https:'; + + let fallbackFn: HttpRequestFunction; + if (options.http2) { + fallbackFn = http2wrapper.auto; + } else { + fallbackFn = isHttps ? https.request : http.request; + } + + const realFn = options.request ?? fallbackFn; + const fn = options.cacheableRequest ? cacheFn : realFn; + + if (agent && !options.http2) { + (options as unknown as RequestOptions).agent = agent[isHttps ? 'https' : 'http']; + } + + options[kRequest] = realFn as HttpRequestFunction; + delete options.request; + delete options.timeout; + + let requestOrResponse: ReturnType; + + try { + requestOrResponse = await fn(url, options as unknown as RequestOptions); + + if (is.undefined(requestOrResponse)) { + requestOrResponse = fallbackFn(url, options as unknown as RequestOptions); + } + + // Restore options + options.request = request; + options.timeout = timeout; + options.agent = agent; + + if (isClientRequest(requestOrResponse)) { + this._onRequest(requestOrResponse); + + // Emit the response after the stream has been ended + } else if (this.writableFinished) { + this._onResponse(requestOrResponse as IncomingMessage); + } else { + this.once('finish', () => { + this._onResponse(requestOrResponse as IncomingMessage); + }); + + this._unlockWrite(); + this.end(); + this._lockWrite(); + } + } catch (error) { + if (error instanceof RequestError) { + throw error; + } + + throw new RequestError(error.message, error, options, this); + } + } + + async _beforeError(error: RequestError): Promise { + try { + const {response} = error; + if (response && is.undefined(response.body)) { + response.body = await getStream.buffer(response, this.options); + } + } catch (_) {} + + try { + for (const hook of this.options.hooks.beforeError) { + // eslint-disable-next-line no-await-in-loop + error = await hook(error); + } + } catch (error_) { + error = error_; + } + + this.destroy(error); + } + + _read(): void { + if (kResponse in this) { + let data; + + while ((data = this[kResponse]!.read()) !== null) { + this[kDownloadedSize] += data.length; + this[kStartedReading] = true; + + const progress = this.downloadProgress; + + if (progress.percent < 1) { + this.emit('downloadProgress', progress); + } + + this.push(data); + } + } + } + + _write(chunk: any, encoding: string, callback: (error?: Error | null) => void): void { + const write = (): void => { + this._writeRequest(chunk, encoding, callback); + }; + + if (this.finalized) { + write(); + } else { + this.once('finalized', write); + } + } + + _writeRequest(chunk: any, encoding: string, callback: (error?: Error | null) => void): void { + this[kRequest]!.write(chunk, encoding, (error?: Error | null) => { + if (!error) { + this[kUploadedSize] += Buffer.byteLength(chunk, encoding as BufferEncoding); + + const progress = this.uploadProgress; + + if (progress.percent < 1) { + this.emit('uploadProgress', progress); + } + } + + callback(error); + }); + } + + _final(callback: (error?: Error | null) => void): void { + const endRequest = (): void => { + // We need to check if `this[kRequest]` is present, + // because it isn't when we use cache. + if (!(kRequest in this)) { + callback(); + return; + } + + this[kRequest]!.end((error?: Error | null) => { + if (!error) { + this[kBodySize] = this[kUploadedSize]; + + this.emit('uploadProgress', this.uploadProgress); + this[kRequest]!.emit('upload-complete'); + } + + callback(error); + }); + }; + + if (this.finalized) { + endRequest(); + } else { + this.once('finalized', endRequest); + } + } + + _destroy(error: Error | null, callback: (error: Error | null) => void): void { + if (kRequest in this) { + this[kRequest]!.abort(); + } else { + this.once('finalized', (): void => { + if (kRequest in this) { + this[kRequest]!.abort(); + } + }); + } + + if (error !== null && !is.undefined(error) && !(error instanceof RequestError)) { + error = new RequestError(error.message, error, this.options); + } + + callback(error); + } + + get ip(): string | undefined { + return this[kRequest]?.socket.remoteAddress; + } + + get aborted(): boolean { + return Boolean(this[kRequest]?.aborted); + } + + get downloadProgress(): Progress { + let percent; + if (this[kResponseSize]) { + percent = this[kDownloadedSize] / this[kResponseSize]!; + } else if (this[kResponseSize] === this[kDownloadedSize]) { + percent = 1; + } else { + percent = 0; + } + + return { + percent, + transferred: this[kDownloadedSize], + total: this[kResponseSize] + }; + } + + get uploadProgress(): Progress { + let percent; + if (this[kBodySize]) { + percent = this[kUploadedSize] / this[kBodySize]!; + } else if (this[kBodySize] === this[kUploadedSize]) { + percent = 1; + } else { + percent = 0; + } + + return { + percent, + transferred: this[kUploadedSize], + total: this[kBodySize] + }; + } + + get timings(): Timings | undefined { + return (this[kRequest] as ClientRequestWithTimings)?.timings; + } + + get isFromCache(): boolean | undefined { + return this[kIsFromCache]; + } + + pipe(destination: T, options?: {end?: boolean}): T { + if (this[kStartedReading]) { + throw new Error('Failed to pipe. The response has been emitted already.'); + } + + if (destination instanceof ServerResponse) { + this[kServerResponsesPiped].add(destination); + } + + return super.pipe(destination, options); + } + + unpipe(destination: T): this { + if (destination instanceof ServerResponse) { + this[kServerResponsesPiped].delete(destination); + } + + super.unpipe(destination); + + return this; + } +} diff --git a/source/create.ts b/source/create.ts index e59a39ee6..f77113e5c 100644 --- a/source/create.ts +++ b/source/create.ts @@ -1,22 +1,50 @@ -import {Merge} from 'type-fest'; -import is from '@sindresorhus/is'; -import asPromise, {createRejection} from './as-promise'; -import asStream, {ProxyStream} from './as-stream'; -import * as errors from './errors'; -import {normalizeArguments, mergeOptions} from './normalize-arguments'; -import deepFreeze from './utils/deep-freeze'; -import { - CancelableRequest, - Defaults, - DefaultOptions, - ExtendOptions, - HandlerFunction, +import {URL} from 'url'; +import {CancelError} from 'p-cancelable'; +import asPromise, { + PromisableRequest, NormalizedOptions, + CancelableRequest, Options, Response, - URLOrOptions, + Defaults as DefaultOptions, + ParseError, PaginationOptions -} from './types'; +} from './as-promise'; +import createRejection from './as-promise/create-rejection'; +import Request, { + RequestError, + CacheError, + ReadError, + HTTPError, + MaxRedirectsError, + TimeoutError, + UnsupportedProtocolError, + UploadError, + kIsNormalizedAlready +} from './core'; +import deepFreeze from './utils/deep-freeze'; + +export interface InstanceDefaults { + options: DefaultOptions; + handlers: HandlerFunction[]; + mutableDefaults: boolean; + _rawHandlers?: HandlerFunction[]; +} + +const errors = { + RequestError, + CacheError, + ReadError, + HTTPError, + MaxRedirectsError, + TimeoutError, + ParseError, + CancelError, + UnsupportedProtocolError, + UploadError +}; + +const {normalizeArguments} = PromisableRequest; export type HTTPAlias = | 'get' @@ -26,70 +54,96 @@ export type HTTPAlias = | 'head' | 'delete'; -export type ReturnStream = (url: string | Merge, options?: Merge) => ProxyStream; -export type GotReturn = CancelableRequest | ProxyStream; +export type GotReturn = Request | CancelableRequest; + +interface GotStreamFunction { + (url: string | URL, options?: Options & {isStream?: true}): Request; + (options?: Options & {isStream?: true}): Request; +} + +const getPromiseOrStream = (options: NormalizedOptions): GotReturn => options.isStream ? new Request(options.url, options) : asPromise(options); -const getPromiseOrStream = (options: NormalizedOptions): GotReturn => options.isStream ? asStream(options) : asPromise(options); +export type HandlerFunction = (options: NormalizedOptions, next: (options: NormalizedOptions) => T) => T | Promise; + +export interface ExtendOptions extends Options { + handlers?: HandlerFunction[]; + mutableDefaults?: boolean; +} const isGotInstance = (value: Got | ExtendOptions): value is Got => ( - Reflect.has(value, 'defaults') && Reflect.has(value.defaults, 'options') + 'defaults' in value && 'options' in value.defaults ); -export type OptionsOfDefaultResponseBody = Merge; -type OptionsOfTextResponseBody = Merge; -type OptionsOfJSONResponseBody = Merge; -type OptionsOfBufferResponseBody = Merge; +type Except = Pick>; + +export type OptionsOfTextResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType?: 'text'}; +export type OptionsOfJSONResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType: 'json'}; +export type OptionsOfBufferResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType: 'buffer'}; +export type GotStrictOptions = Except; type ResponseBodyOnly = {resolveBodyOnly: true}; -/** -Can be used to match methods explicitly or parameters extraction: `Parameters`. -*/ -export interface GotRequestMethod { +export interface GotPaginate { + (url: string | URL, options?: Options & PaginationOptions): AsyncIterableIterator; + all(url: string | URL, options?: Options & PaginationOptions): Promise; + + // A bug. + // eslint-disable-next-line @typescript-eslint/adjacent-overload-signatures + (options?: Options & PaginationOptions): AsyncIterableIterator; + // A bug. + // eslint-disable-next-line @typescript-eslint/adjacent-overload-signatures + all(options?: Options & PaginationOptions): Promise; +} + +export interface GotRequest { // `asPromise` usage - (url: string | OptionsOfDefaultResponseBody, options?: OptionsOfDefaultResponseBody): CancelableRequest>; - (url: string | OptionsOfTextResponseBody, options?: OptionsOfTextResponseBody): CancelableRequest>; - (url: string | OptionsOfJSONResponseBody, options?: OptionsOfJSONResponseBody): CancelableRequest>; - (url: string | OptionsOfBufferResponseBody, options?: OptionsOfBufferResponseBody): CancelableRequest>; + (url: string | URL, options?: OptionsOfTextResponseBody): CancelableRequest>; + (url: string | URL, options?: OptionsOfJSONResponseBody): CancelableRequest>; + (url: string | URL, options?: OptionsOfBufferResponseBody): CancelableRequest>; + + (options: OptionsOfTextResponseBody): CancelableRequest>; + (options: OptionsOfJSONResponseBody): CancelableRequest>; + (options: OptionsOfBufferResponseBody): CancelableRequest>; // `resolveBodyOnly` usage - (url: string | Merge, options?: Merge): CancelableRequest; - (url: string | Merge, options?: Merge): CancelableRequest; - (url: string | Merge, options?: Merge): CancelableRequest; - (url: string | Merge, options?: Merge): CancelableRequest; + (url: string | URL, options?: (OptionsOfTextResponseBody & ResponseBodyOnly)): CancelableRequest; + (url: string | URL, options?: (OptionsOfJSONResponseBody & ResponseBodyOnly)): CancelableRequest; + (url: string | URL, options?: (OptionsOfBufferResponseBody & ResponseBodyOnly)): CancelableRequest; + + (options: (OptionsOfTextResponseBody & ResponseBodyOnly)): CancelableRequest; + (options: (OptionsOfJSONResponseBody & ResponseBodyOnly)): CancelableRequest; + (options: (OptionsOfBufferResponseBody & ResponseBodyOnly)): CancelableRequest; // `asStream` usage - (url: string | Merge, options?: Merge): ProxyStream; -} + (url: string | URL, options?: Options & {isStream: true}): Request; -export interface GotPaginate { - (url: URLOrOptions & PaginationOptions, options?: Options & PaginationOptions): AsyncIterableIterator; - all(url: URLOrOptions & PaginationOptions, options?: Options & PaginationOptions): Promise; + (options: Options & {isStream: true}): Request; + + // Fallback + (url: string | URL, options?: Options): CancelableRequest | Request; + + (options: Options): CancelableRequest | Request; } -export interface Got extends Record, GotRequestMethod { +export type GotStream = GotStreamFunction & Record; + +export interface Got extends Record, GotRequest { stream: GotStream; paginate: GotPaginate; - defaults: Defaults; - GotError: typeof errors.GotError; - CacheError: typeof errors.CacheError; - RequestError: typeof errors.RequestError; - ReadError: typeof errors.ReadError; - ParseError: typeof errors.ParseError; - HTTPError: typeof errors.HTTPError; - MaxRedirectsError: typeof errors.MaxRedirectsError; - UnsupportedProtocolError: typeof errors.UnsupportedProtocolError; - TimeoutError: typeof errors.TimeoutError; - CancelError: typeof errors.CancelError; + defaults: InstanceDefaults; + CacheError: typeof CacheError; + RequestError: typeof RequestError; + ReadError: typeof ReadError; + ParseError: typeof ParseError; + HTTPError: typeof HTTPError; + MaxRedirectsError: typeof MaxRedirectsError; + TimeoutError: typeof TimeoutError; + CancelError: typeof CancelError; extend(...instancesOrOptions: Array): Got; mergeInstances(parent: Got, ...instances: Got[]): Got; mergeOptions(...sources: Options[]): NormalizedOptions; } -export interface GotStream extends Record { - (url: URLOrOptions, options?: Options): ProxyStream; -} - const aliases: readonly HTTPAlias[] = [ 'get', 'post', @@ -101,7 +155,17 @@ const aliases: readonly HTTPAlias[] = [ export const defaultHandler: HandlerFunction = (options, next) => next(options); -const create = (defaults: Defaults): Got => { +export const mergeOptions = (...sources: Options[]): NormalizedOptions => { + let mergedOptions: NormalizedOptions | undefined; + + for (const source of sources) { + mergedOptions = normalizeArguments(undefined, source, mergedOptions); + } + + return mergedOptions!; +}; + +const create = (defaults: InstanceDefaults): Got => { // Proxy properties from next handlers defaults._rawHandlers = defaults.handlers; defaults.handlers = defaults.handlers.map(fn => ((options, next) => { @@ -130,8 +194,7 @@ const create = (defaults: Defaults): Got => { return result; })); - // @ts-ignore Because the for loop handles it for us, as well as the other Object.defines - const got: Got = (url: URLOrOptions, options?: Options): GotReturn => { + const got: Got = ((url: string | URL, options?: Options): GotReturn => { let iteration = 0; const iterateHandlers = (newOptions: NormalizedOptions): GotReturn => { return defaults.handlers[iteration++]( @@ -140,19 +203,23 @@ const create = (defaults: Defaults): Got => { ) as GotReturn; }; - /* eslint-disable @typescript-eslint/return-await */ try { - return iterateHandlers(normalizeArguments(url, options, defaults)); + const normalizedOptions = normalizeArguments(url, options, defaults.options); + normalizedOptions[kIsNormalizedAlready] = true; + + // A bug. + // eslint-disable-next-line @typescript-eslint/return-await + return iterateHandlers(normalizedOptions); } catch (error) { if (options?.isStream) { throw error; } else { - // @ts-ignore It's an Error not a response, but TS thinks it's calling .resolve + // A bug. + // eslint-disable-next-line @typescript-eslint/return-await return createRejection(error); } } - /* eslint-enable @typescript-eslint/return-await */ - }; + }) as Got; got.extend = (...instancesOrOptions) => { const optionsArray: Options[] = [defaults.options]; @@ -167,8 +234,8 @@ const create = (defaults: Defaults): Got => { } else { optionsArray.push(value); - if (Reflect.has(value, 'handlers')) { - handlers.push(...value.handlers); + if ('handlers' in value) { + handlers.push(...value.handlers!); } isMutableDefaults = value.mutableDefaults; @@ -182,44 +249,34 @@ const create = (defaults: Defaults): Got => { } return create({ - options: mergeOptions(...optionsArray) as DefaultOptions, + options: mergeOptions(...optionsArray), handlers, mutableDefaults: Boolean(isMutableDefaults) }); }; - // @ts-ignore The missing methods because the for-loop handles it for us - got.stream = (url, options) => got(url, {...options, isStream: true}); - - for (const method of aliases) { - // @ts-ignore Cannot properly type a function with multiple definitions yet - got[method] = (url: URLOrOptions, options?: Options): GotReturn => got(url, {...options, method}); - got.stream[method] = (url, options) => got.stream(url, {...options, method}); - } - - // @ts-ignore The missing property is added below - got.paginate = async function * (url: URLOrOptions, options?: Options) { - let normalizedOptions = normalizeArguments(url, options, defaults); + got.paginate = (async function * (url: string | URL, options?: Options) { + let normalizedOptions = normalizeArguments(url, options, defaults.options); const pagination = normalizedOptions._pagination!; - if (!is.object(pagination)) { - throw new Error('`options._pagination` must be implemented'); + if (typeof pagination !== 'object') { + throw new TypeError('`options._pagination` must be implemented'); } const all: T[] = []; while (true) { - // @ts-ignore See https://github.com/sindresorhus/got/issues/954 + // TODO: Throw when result is not an instance of Response // eslint-disable-next-line no-await-in-loop - const result = await got(normalizedOptions); + const result = (await got('', normalizedOptions)) as Response; // eslint-disable-next-line no-await-in-loop - const parsed = await pagination.transform!(result); + const parsed = await pagination.transform(result); for (const item of parsed) { - if (pagination.filter!(item, all)) { - if (!pagination.shouldContinue!(item, all)) { + if (pagination.filter(item, all)) { + if (!pagination.shouldContinue(item, all)) { return; } @@ -233,19 +290,19 @@ const create = (defaults: Defaults): Got => { } } - const optionsToMerge = pagination.paginate!(result); + const optionsToMerge = pagination.paginate(result); if (optionsToMerge === false) { return; } if (optionsToMerge !== undefined) { - normalizedOptions = normalizeArguments(normalizedOptions, optionsToMerge); + normalizedOptions = normalizeArguments(undefined, optionsToMerge, normalizedOptions); } } - }; + }) as GotPaginate; - got.paginate.all = async (url: URLOrOptions, options?: Options) => { + got.paginate.all = (async (url: string | URL, options?: Options) => { const results: T[] = []; for await (const item of got.paginate(url, options)) { @@ -253,7 +310,17 @@ const create = (defaults: Defaults): Got => { } return results; - }; + }) as GotPaginate['all']; + + got.stream = ((url: string | URL, options?: Options) => got(url, {...options, isStream: true})) as GotStream; + + for (const method of aliases) { + got[method] = ((url: string | URL, options?: Options): GotReturn => got(url, {...options, method})) as GotRequest; + + got.stream[method] = ((url: string | URL, options?: Options & {isStream: true}) => { + return got(url, {...options, method, isStream: true}); + }) as GotStream; + } Object.assign(got, {...errors, mergeOptions}); Object.defineProperty(got, 'defaults', { diff --git a/source/errors.ts b/source/errors.ts deleted file mode 100644 index ec62394a0..000000000 --- a/source/errors.ts +++ /dev/null @@ -1,125 +0,0 @@ -import is from '@sindresorhus/is'; -import {Timings} from '@szmarczak/http-timer'; -import {TimeoutError as TimedOutError} from './utils/timed-out'; -import {Response, NormalizedOptions} from './types'; - -export class GotError extends Error { - code?: string; - stack!: string; - declare readonly options: NormalizedOptions; - - constructor(message: string, error: Partial, options: NormalizedOptions) { - super(message); - Error.captureStackTrace(this, this.constructor); - this.name = 'GotError'; - - if (!is.undefined(error.code)) { - this.code = error.code; - } - - Object.defineProperty(this, 'options', { - // This fails because of TS 3.7.2 useDefineForClassFields - // Ref: https://github.com/microsoft/TypeScript/issues/34972 - enumerable: false, - value: options - }); - - // Recover the original stacktrace - if (!is.undefined(error.stack)) { - const indexOfMessage = this.stack.indexOf(this.message) + this.message.length; - const thisStackTrace = this.stack.slice(indexOfMessage).split('\n').reverse(); - const errorStackTrace = error.stack.slice(error.stack.indexOf(error.message!) + error.message!.length).split('\n').reverse(); - - // Remove duplicated traces - while (errorStackTrace.length !== 0 && errorStackTrace[0] === thisStackTrace[0]) { - thisStackTrace.shift(); - } - - this.stack = `${this.stack.slice(0, indexOfMessage)}${thisStackTrace.reverse().join('\n')}${errorStackTrace.reverse().join('\n')}`; - } - } -} - -export class CacheError extends GotError { - constructor(error: Error, options: NormalizedOptions) { - super(error.message, error, options); - this.name = 'CacheError'; - } -} - -export class RequestError extends GotError { - constructor(error: Error, options: NormalizedOptions) { - super(error.message, error, options); - this.name = 'RequestError'; - } -} - -export class ReadError extends GotError { - constructor(error: Error, options: NormalizedOptions) { - super(error.message, error, options); - this.name = 'ReadError'; - } -} - -export class ParseError extends GotError { - declare readonly response: Response; - - constructor(error: Error, response: Response, options: NormalizedOptions) { - super(`${error.message} in "${options.url.toString()}"`, error, options); - this.name = 'ParseError'; - - Object.defineProperty(this, 'response', { - enumerable: false, - value: response - }); - } -} - -export class HTTPError extends GotError { - declare readonly response: Response; - - constructor(response: Response, options: NormalizedOptions) { - super(`Response code ${response.statusCode} (${response.statusMessage!})`, {}, options); - this.name = 'HTTPError'; - - Object.defineProperty(this, 'response', { - enumerable: false, - value: response - }); - } -} - -export class MaxRedirectsError extends GotError { - declare readonly response: Response; - - constructor(response: Response, maxRedirects: number, options: NormalizedOptions) { - super(`Redirected ${maxRedirects} times. Aborting.`, {}, options); - this.name = 'MaxRedirectsError'; - - Object.defineProperty(this, 'response', { - enumerable: false, - value: response - }); - } -} - -export class UnsupportedProtocolError extends GotError { - constructor(options: NormalizedOptions) { - super(`Unsupported protocol "${options.url.protocol}"`, {}, options); - this.name = 'UnsupportedProtocolError'; - } -} - -export class TimeoutError extends GotError { - timings: Timings; - event: string; - - constructor(error: TimedOutError, timings: Timings, options: NormalizedOptions) { - super(error.message, error, options); - this.name = 'TimeoutError'; - this.event = error.event; - this.timings = timings; - } -} - -export {CancelError} from 'p-cancelable'; diff --git a/source/get-response.ts b/source/get-response.ts deleted file mode 100644 index bc1ceb394..000000000 --- a/source/get-response.ts +++ /dev/null @@ -1,37 +0,0 @@ -import decompressResponse = require('decompress-response'); -import EventEmitter = require('events'); -import mimicResponse = require('mimic-response'); -import stream = require('stream'); -import {IncomingMessage} from 'http'; -import {promisify} from 'util'; -import {createProgressStream} from './progress'; -import {NormalizedOptions} from './types'; - -const pipeline = promisify(stream.pipeline); - -export default async (response: IncomingMessage, options: NormalizedOptions, emitter: EventEmitter): Promise => { - const downloadBodySize = Number(response.headers['content-length']) || undefined; - const progressStream = createProgressStream('downloadProgress', emitter, downloadBodySize); - - mimicResponse(response, progressStream); - - const newResponse = ( - options.decompress && - options.method !== 'HEAD' ? decompressResponse(progressStream as unknown as IncomingMessage) : progressStream - ) as IncomingMessage; - - if (!options.decompress && ['gzip', 'deflate', 'br'].includes(newResponse.headers['content-encoding'] ?? '')) { - options.responseType = 'buffer'; - } - - emitter.emit('response', newResponse); - - return pipeline( - response, - progressStream - ).catch(error => { - if (error.code !== 'ERR_STREAM_PREMATURE_CLOSE') { - throw error; - } - }); -}; diff --git a/source/index.ts b/source/index.ts index 1dbf44679..424ea6f5c 100644 --- a/source/index.ts +++ b/source/index.ts @@ -1,8 +1,8 @@ import {URL} from 'url'; -import create, {defaultHandler} from './create'; -import {Defaults, Response, GotOptions} from './types'; +import {Response, Options} from './as-promise'; +import create, {defaultHandler, InstanceDefaults} from './create'; -const defaults: Defaults = { +const defaults: InstanceDefaults = { options: { method: 'GET', retry: { @@ -56,17 +56,16 @@ const defaults: Defaults = { throwHttpErrors: true, followRedirect: true, isStream: false, - cache: false, - dnsCache: false, - useElectronNet: false, responseType: 'text', resolveBodyOnly: false, maxRedirects: 10, prefixUrl: '', methodRewriting: true, - allowGetBody: false, ignoreInvalidCookies: false, context: {}, + http2: false, + allowGetBody: false, + rejectUnauthorized: true, _pagination: { transform: (response: Response) => { return JSON.parse(response.body as string); @@ -90,7 +89,7 @@ const defaults: Defaults = { } if (next) { - const options: GotOptions = { + const options: Options = { url: new URL(next) }; @@ -116,42 +115,5 @@ export default got; module.exports = got; module.exports.default = got; -// Export types -export * from './types'; - -export { - Got, - GotStream, - ReturnStream, - GotRequestMethod, - GotReturn -} from './create'; - -export { - ProxyStream as ResponseStream -} from './as-stream'; - -export { - GotError, - CacheError, - RequestError, - ReadError, - ParseError, - HTTPError, - MaxRedirectsError, - UnsupportedProtocolError, - TimeoutError, - CancelError -} from './errors'; - -export { - InitHook, - BeforeRequestHook, - BeforeRedirectHook, - BeforeRetryHook, - BeforeErrorHook, - AfterResponseHook, - HookType, - Hooks, - HookEvent -} from './known-hook-events'; +export * from './create'; +export * from './as-promise'; diff --git a/source/known-hook-events.ts b/source/known-hook-events.ts deleted file mode 100644 index 08120857b..000000000 --- a/source/known-hook-events.ts +++ /dev/null @@ -1,117 +0,0 @@ -import {CancelableRequest, GeneralError, NormalizedOptions, Options, Response} from './types'; - -/** -Called with plain request options, right before their normalization. This is especially useful in conjunction with `got.extend()` when the input needs custom handling. - -**Note:** This hook must be synchronous. - -@see [Request migration guide](https://github.com/sindresorhus/got/blob/master/migration-guides.md#breaking-changes) for an example. -*/ -export type InitHook = (options: Options) => void; - -/** -Called with normalized [request options](https://github.com/sindresorhus/got#options). Got will make no further changes to the request before it is sent (except the body serialization). This is especially useful in conjunction with [`got.extend()`](https://github.com/sindresorhus/got#instances) when you want to create an API client that, for example, uses HMAC-signing. - -@see [AWS section](https://github.com/sindresorhus/got#aws) for an example. -*/ -export type BeforeRequestHook = (options: NormalizedOptions) => void | Promise; - -/** -Called with normalized [request options](https://github.com/sindresorhus/got#options). Got will make no further changes to the request. This is especially useful when you want to avoid dead sites. -*/ -export type BeforeRedirectHook = (options: NormalizedOptions, response: Response) => void | Promise; - -/** -Called with normalized [request options](https://github.com/sindresorhus/got#options), the error and the retry count. Got will make no further changes to the request. This is especially useful when some extra work is required before the next try. -*/ -export type BeforeRetryHook = (options: NormalizedOptions, error?: GeneralError, retryCount?: number) => void | Promise; - -/** -Called with an `Error` instance. The error is passed to the hook right before it's thrown. This is especially useful when you want to have more detailed errors. - -**Note:** Errors thrown while normalizing input options are thrown directly and not part of this hook. -*/ -export type BeforeErrorHook = (error: ErrorLike) => GeneralError | Promise; - -/** -Called with [response object](https://github.com/sindresorhus/got#response) and a retry function. - -Each function should return the response. This is especially useful when you want to refresh an access token. -*/ -export type AfterResponseHook = (response: Response, retryWithMergedOptions: (options: Options) => CancelableRequest) => Response | CancelableRequest | Promise>; - -export type HookType = - | BeforeErrorHook - | InitHook - | BeforeRequestHook - | BeforeRedirectHook - | BeforeRetryHook - | AfterResponseHook; - -/** -Hooks allow modifications during the request lifecycle. Hook functions may be async and are run serially. -*/ -export interface Hooks { - /** - Called with plain request options, right before their normalization. This is especially useful in conjunction with `got.extend()` when the input needs custom handling. - - **Note:** This hook must be synchronous. - - @see [Request migration guide](https://github.com/sindresorhus/got/blob/master/migration-guides.md#breaking-changes) for an example. - @default [] - */ - init?: InitHook[]; - - /** - Called with normalized [request options](https://github.com/sindresorhus/got#options). Got will make no further changes to the request before it is sent (except the body serialization). This is especially useful in conjunction with [`got.extend()`](https://github.com/sindresorhus/got#instances) when you want to create an API client that, for example, uses HMAC-signing. - - @see [AWS section](https://github.com/sindresorhus/got#aws) for an example. - @default [] - */ - beforeRequest?: BeforeRequestHook[]; - - /** - Called with normalized [request options](https://github.com/sindresorhus/got#options). Got will make no further changes to the request. This is especially useful when you want to avoid dead sites. - - @default [] - */ - beforeRedirect?: BeforeRedirectHook[]; - - /** - Called with normalized [request options](https://github.com/sindresorhus/got#options), the error and the retry count. Got will make no further changes to the request. This is especially useful when some extra work is required before the next try. - - @default [] - */ - beforeRetry?: BeforeRetryHook[]; - - /** - Called with an `Error` instance. The error is passed to the hook right before it's thrown. This is especially useful when you want to have more detailed errors. - - **Note:** Errors thrown while normalizing input options are thrown directly and not part of this hook. - - @default [] - */ - beforeError?: BeforeErrorHook[]; - - /** - Called with [response object](https://github.com/sindresorhus/got#response) and a retry function. - - Each function should return the response. This is especially useful when you want to refresh an access token. - - @default [] - */ - afterResponse?: AfterResponseHook[]; -} - -export type HookEvent = keyof Hooks; - -const knownHookEvents: readonly HookEvent[] = [ - 'beforeError', - 'init', - 'beforeRequest', - 'beforeRedirect', - 'beforeRetry', - 'afterResponse' -]; - -export default knownHookEvents; diff --git a/source/normalize-arguments.ts b/source/normalize-arguments.ts deleted file mode 100644 index 831b7981c..000000000 --- a/source/normalize-arguments.ts +++ /dev/null @@ -1,536 +0,0 @@ -import {URL, URLSearchParams} from 'url'; -import {promisify, deprecate} from 'util'; -import CacheableRequest = require('cacheable-request'); -import http = require('http'); -import https = require('https'); -import Keyv = require('keyv'); -import lowercaseKeys = require('lowercase-keys'); -import stream = require('stream'); -import toReadableStream = require('to-readable-stream'); -import is from '@sindresorhus/is'; -import CacheableLookup from 'cacheable-lookup'; -import {Merge} from 'type-fest'; -import {UnsupportedProtocolError} from './errors'; -import knownHookEvents, {InitHook} from './known-hook-events'; -import dynamicRequire from './utils/dynamic-require'; -import getBodySize from './utils/get-body-size'; -import isFormData from './utils/is-form-data'; -import merge from './utils/merge'; -import optionsToUrl from './utils/options-to-url'; -import supportsBrotli from './utils/supports-brotli'; -import { - AgentByProtocol, - Defaults, - Method, - NormalizedOptions, - Options, - RequestFunction, - URLOrOptions, - requestSymbol -} from './types'; - -// `preNormalizeArguments` normalizes these options: `headers`, `prefixUrl`, `hooks`, `timeout`, `retry` and `method`. -// `normalizeArguments` is *only* called on `got(...)`. It normalizes the URL and performs `mergeOptions(...)`. -// `normalizeRequestArguments` converts Got options into HTTP options. - -type NonEnumerableProperty = 'context' | 'body' | 'json' | 'form'; -const nonEnumerableProperties: NonEnumerableProperty[] = [ - 'context', - 'body', - 'json', - 'form' -]; - -const isAgentByProtocol = (agent: Options['agent']): agent is AgentByProtocol => is.object(agent); - -// TODO: `preNormalizeArguments` should merge `options` & `defaults` -export const preNormalizeArguments = (options: Options, defaults?: NormalizedOptions): NormalizedOptions => { - // `options.headers` - if (is.undefined(options.headers)) { - options.headers = {}; - } else { - options.headers = lowercaseKeys(options.headers); - } - - for (const [key, value] of Object.entries(options.headers)) { - if (is.null_(value)) { - throw new TypeError(`Use \`undefined\` instead of \`null\` to delete the \`${key}\` header`); - } - } - - // `options.prefixUrl` - if (is.urlInstance(options.prefixUrl) || is.string(options.prefixUrl)) { - options.prefixUrl = options.prefixUrl.toString(); - - if (options.prefixUrl.length !== 0 && !options.prefixUrl.endsWith('/')) { - options.prefixUrl += '/'; - } - } else { - options.prefixUrl = defaults ? defaults.prefixUrl : ''; - } - - // `options.hooks` - if (is.undefined(options.hooks)) { - options.hooks = {}; - } - - if (is.object(options.hooks)) { - for (const event of knownHookEvents) { - if (Reflect.has(options.hooks, event)) { - if (!is.array(options.hooks[event])) { - throw new TypeError(`Parameter \`${event}\` must be an Array, not ${is(options.hooks[event])}`); - } - } else { - options.hooks[event] = []; - } - } - } else { - throw new TypeError(`Parameter \`hooks\` must be an Object, not ${is(options.hooks)}`); - } - - if (defaults) { - for (const event of knownHookEvents) { - if (!(Reflect.has(options.hooks, event) && is.undefined(options.hooks[event]))) { - // @ts-ignore Union type array is not assignable to union array type - options.hooks[event] = [ - ...defaults.hooks[event], - ...options.hooks[event]! - ]; - } - } - } - - // `options.timeout` - if (is.number(options.timeout)) { - options.timeout = {request: options.timeout}; - } else if (!is.object(options.timeout)) { - options.timeout = {}; - } - - // `options.retry` - const {retry} = options; - - if (defaults) { - options.retry = {...defaults.retry}; - } else { - options.retry = { - calculateDelay: retryObject => retryObject.computedValue, - limit: 0, - methods: [], - statusCodes: [], - errorCodes: [], - maxRetryAfter: undefined - }; - } - - if (is.object(retry)) { - options.retry = { - ...options.retry, - ...retry - }; - } else if (is.number(retry)) { - options.retry.limit = retry; - } - - if (options.retry.maxRetryAfter === undefined) { - options.retry.maxRetryAfter = Math.min( - ...[options.timeout.request, options.timeout.connect].filter((n): n is number => !is.nullOrUndefined(n)) - ); - } - - options.retry.methods = [...new Set(options.retry.methods!.map(method => method.toUpperCase() as Method))]; - options.retry.statusCodes = [...new Set(options.retry.statusCodes)]; - options.retry.errorCodes = [...new Set(options.retry.errorCodes)]; - - // `options.dnsCache` - if (options.dnsCache && !(options.dnsCache instanceof CacheableLookup)) { - options.dnsCache = new CacheableLookup({cacheAdapter: options.dnsCache as Keyv}); - } - - // `options.method` - if (is.string(options.method)) { - options.method = options.method.toUpperCase() as Method; - } else { - options.method = defaults?.method ?? 'GET'; - } - - // Better memory management, so we don't have to generate a new object every time - if (options.cache) { - (options as NormalizedOptions).cacheableRequest = new CacheableRequest( - // @ts-ignore Cannot properly type a function with multiple definitions yet - (requestOptions, handler) => requestOptions[requestSymbol](requestOptions, handler), - options.cache - ); - } - - // `options.cookieJar` - if (is.object(options.cookieJar)) { - let {setCookie, getCookieString} = options.cookieJar; - - // Horrible `tough-cookie` check - if (setCookie.length === 4 && getCookieString.length === 0) { - if (!Reflect.has(setCookie, promisify.custom)) { - // @ts-ignore TS is dumb - it says `setCookie` is `never`. - setCookie = promisify(setCookie.bind(options.cookieJar)); - getCookieString = promisify(getCookieString.bind(options.cookieJar)); - } - } else if (setCookie.length !== 2) { - throw new TypeError('`options.cookieJar.setCookie` needs to be an async function with 2 arguments'); - } else if (getCookieString.length !== 1) { - throw new TypeError('`options.cookieJar.getCookieString` needs to be an async function with 1 argument'); - } - - options.cookieJar = {setCookie, getCookieString}; - } - - // `options.encoding` - if (is.null_(options.encoding)) { - throw new TypeError('To get a Buffer, set `options.responseType` to `buffer` instead'); - } - - // `options.maxRedirects` - if (!Reflect.has(options, 'maxRedirects') && !(defaults && Reflect.has(defaults, 'maxRedirects'))) { - options.maxRedirects = 0; - } - - // Merge defaults - if (defaults) { - options = merge({}, defaults, options); - } - - // `options._pagination` - if (is.object(options._pagination)) { - const {_pagination: pagination} = options; - - if (!is.function_(pagination.transform)) { - throw new TypeError('`options._pagination.transform` must be implemented'); - } - - if (!is.function_(pagination.shouldContinue)) { - throw new TypeError('`options._pagination.shouldContinue` must be implemented'); - } - - if (!is.function_(pagination.filter)) { - throw new TypeError('`options._pagination.filter` must be implemented'); - } - - if (!is.function_(pagination.paginate)) { - throw new TypeError('`options._pagination.paginate` must be implemented'); - } - } - - // Other values - options.decompress = Boolean(options.decompress); - options.isStream = Boolean(options.isStream); - options.throwHttpErrors = Boolean(options.throwHttpErrors); - options.ignoreInvalidCookies = Boolean(options.ignoreInvalidCookies); - options.cache = options.cache ?? false; - options.responseType = options.responseType ?? 'text'; - options.resolveBodyOnly = Boolean(options.resolveBodyOnly); - options.followRedirect = Boolean(options.followRedirect); - options.dnsCache = options.dnsCache ?? false; - options.useElectronNet = Boolean(options.useElectronNet); - options.methodRewriting = Boolean(options.methodRewriting); - options.allowGetBody = Boolean(options.allowGetBody); - options.context = options.context ?? {}; - - return options as NormalizedOptions; -}; - -export const mergeOptions = (...sources: Options[]): NormalizedOptions => { - let mergedOptions = preNormalizeArguments({}); - - // Non enumerable properties shall not be merged - const properties: Partial<{[Key in NonEnumerableProperty]: any}> = {}; - - for (const source of sources) { - mergedOptions = preNormalizeArguments(merge({}, source), mergedOptions); - - for (const name of nonEnumerableProperties) { - if (!Reflect.has(source, name)) { - continue; - } - - properties[name] = { - writable: true, - configurable: true, - enumerable: false, - value: source[name] - }; - } - } - - Object.defineProperties(mergedOptions, properties); - - return mergedOptions; -}; - -export const normalizeArguments = (url: URLOrOptions, options?: Options, defaults?: Defaults): NormalizedOptions => { - // Merge options - if (typeof url === 'undefined') { - throw new TypeError('Missing `url` argument'); - } - - const runInitHooks = (hooks?: InitHook[], options?: Options): void => { - if (hooks && options) { - for (const hook of hooks) { - const result = hook(options); - - if (is.promise(result)) { - throw new TypeError('The `init` hook must be a synchronous function'); - } - } - } - }; - - const hasUrl = is.urlInstance(url) || is.string(url); - if (hasUrl) { - if (options) { - if (Reflect.has(options, 'url')) { - throw new TypeError('The `url` option cannot be used if the input is a valid URL.'); - } - } else { - options = {}; - } - - // @ts-ignore URL is not URL - options.url = url; - - runInitHooks(defaults?.options.hooks.init, options); - runInitHooks(options.hooks?.init, options); - } else if (Reflect.has(url as object, 'resolve')) { - throw new Error('The legacy `url.Url` is deprecated. Use `URL` instead.'); - } else { - runInitHooks(defaults?.options.hooks.init, url as Options); - runInitHooks((url as Options).hooks?.init, url as Options); - - if (options) { - runInitHooks(defaults?.options.hooks.init, options); - runInitHooks(options.hooks?.init, options); - } - } - - if (hasUrl) { - options = mergeOptions(defaults?.options ?? {}, options ?? {}); - } else { - options = mergeOptions(defaults?.options ?? {}, url as object, options ?? {}); - } - - // Normalize URL - // TODO: drop `optionsToUrl` in Got 12 - if (is.string(options.url)) { - options.url = (options.prefixUrl as string) + options.url; - options.url = options.url.replace(/^unix:/, 'http://$&'); - - if (options.searchParams || options.search) { - options.url = options.url.split('?')[0]; - } - - // @ts-ignore URL is not URL - options.url = optionsToUrl({ - origin: options.url, - ...options - }); - } else if (!is.urlInstance(options.url)) { - // @ts-ignore URL is not URL - options.url = optionsToUrl({origin: options.prefixUrl as string, ...options}); - } - - const normalizedOptions = options as NormalizedOptions; - - // Make it possible to change `options.prefixUrl` - let prefixUrl = options.prefixUrl as string; - Object.defineProperty(normalizedOptions, 'prefixUrl', { - set: (value: string) => { - if (!normalizedOptions.url.href.startsWith(value)) { - throw new Error(`Cannot change \`prefixUrl\` from ${prefixUrl} to ${value}: ${normalizedOptions.url.href}`); - } - - normalizedOptions.url = new URL(value + normalizedOptions.url.href.slice(prefixUrl.length)); - prefixUrl = value; - }, - get: () => prefixUrl - }); - - // Make it possible to remove default headers - for (const [key, value] of Object.entries(normalizedOptions.headers)) { - if (is.undefined(value)) { - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete normalizedOptions.headers[key]; - } - } - - return normalizedOptions; -}; - -const withoutBody: ReadonlySet = new Set(['HEAD']); -const withoutBodyUnlessSpecified = 'GET'; - -export type NormalizedRequestArguments = Merge; -}>; - -export const normalizeRequestArguments = async (options: NormalizedOptions): Promise => { - options = mergeOptions(options); - - // Serialize body - const {headers} = options; - const hasNoContentType = is.undefined(headers['content-type']); - - { - // TODO: these checks should be moved to `preNormalizeArguments` - const isForm = !is.undefined(options.form); - const isJson = !is.undefined(options.json); - const isBody = !is.undefined(options.body); - if ((isBody || isForm || isJson) && withoutBody.has(options.method)) { - throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); - } - - if (!options.allowGetBody && (isBody || isForm || isJson) && withoutBodyUnlessSpecified === options.method) { - throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); - } - - if ([isBody, isForm, isJson].filter(isTrue => isTrue).length > 1) { - throw new TypeError('The `body`, `json` and `form` options are mutually exclusive'); - } - - if ( - isBody && - !is.nodeStream(options.body) && - !is.string(options.body) && - !is.buffer(options.body) && - !(is.object(options.body) && isFormData(options.body)) - ) { - throw new TypeError('The `body` option must be a stream.Readable, string or Buffer'); - } - - if (isForm && !is.object(options.form)) { - throw new TypeError('The `form` option must be an Object'); - } - } - - if (options.body) { - // Special case for https://github.com/form-data/form-data - if (is.object(options.body) && isFormData(options.body) && hasNoContentType) { - headers['content-type'] = `multipart/form-data; boundary=${options.body.getBoundary()}`; - } - } else if (options.form) { - if (hasNoContentType) { - headers['content-type'] = 'application/x-www-form-urlencoded'; - } - - options.body = (new URLSearchParams(options.form as Record)).toString(); - } else if (options.json) { - if (hasNoContentType) { - headers['content-type'] = 'application/json'; - } - - options.body = JSON.stringify(options.json); - } - - const uploadBodySize = await getBodySize(options); - - if (!is.nodeStream(options.body)) { - options.body = toReadableStream(options.body!); - } - - // See https://tools.ietf.org/html/rfc7230#section-3.3.2 - // A user agent SHOULD send a Content-Length in a request message when - // no Transfer-Encoding is sent and the request method defines a meaning - // for an enclosed payload body. For example, a Content-Length header - // field is normally sent in a POST request even when the value is 0 - // (indicating an empty payload body). A user agent SHOULD NOT send a - // Content-Length header field when the request message does not contain - // a payload body and the method semantics do not anticipate such a - // body. - if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding'])) { - if ( - (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH' || options.method === 'DELETE' || (options.allowGetBody && options.method === 'GET')) && - !is.undefined(uploadBodySize) - ) { - // @ts-ignore We assign if it is undefined, so this IS correct - headers['content-length'] = String(uploadBodySize); - } - } - - if (!options.isStream && options.responseType === 'json' && is.undefined(headers.accept)) { - headers.accept = 'application/json'; - } - - if (options.decompress && is.undefined(headers['accept-encoding'])) { - headers['accept-encoding'] = supportsBrotli ? 'gzip, deflate, br' : 'gzip, deflate'; - } - - // Validate URL - if (options.url.protocol !== 'http:' && options.url.protocol !== 'https:') { - throw new UnsupportedProtocolError(options); - } - - decodeURI(options.url.toString()); - - // Normalize request function - if (is.function_(options.request)) { - options[requestSymbol] = options.request; - delete options.request; - } else { - options[requestSymbol] = options.url.protocol === 'https:' ? https.request : http.request; - } - - // UNIX sockets - if (options.url.hostname === 'unix') { - const matches = /(?.+?):(?.+)/.exec(options.url.pathname); - - if (matches?.groups) { - const {socketPath, path} = matches.groups; - - options = { - ...options, - socketPath, - path, - host: '' - }; - } - } - - if (isAgentByProtocol(options.agent)) { - options.agent = options.agent[options.url.protocol.slice(0, -1) as keyof AgentByProtocol] ?? options.agent; - } - - if (options.dnsCache) { - options.lookup = options.dnsCache.lookup; - } - - /* istanbul ignore next: electron.net is broken */ - // No point in typing process.versions correctly, as - // `process.version.electron` is used only once, right here. - if (options.useElectronNet && (process.versions as any).electron) { - const electron = dynamicRequire(module, 'electron') as any; // Trick webpack - options.request = deprecate( - electron.net.request ?? electron.remote.net.request, - 'Electron support has been deprecated and will be removed in Got 11.\n' + - 'See https://github.com/sindresorhus/got/issues/899 for further information.', - 'GOT_ELECTRON' - ); - } - - // Got's `timeout` is an object, http's `timeout` is a number, so they're not compatible. - delete options.timeout; - - // Set cookies - if (options.cookieJar) { - const cookieString = await options.cookieJar.getCookieString(options.url.toString()); - - if (is.nonEmptyString(cookieString)) { - options.headers.cookie = cookieString; - } else { - delete options.headers.cookie; - } - } - - // `http-cache-semantics` checks this - delete options.url; - - return options as unknown as NormalizedRequestArguments; -}; diff --git a/source/progress.ts b/source/progress.ts deleted file mode 100644 index ec04e24e7..000000000 --- a/source/progress.ts +++ /dev/null @@ -1,48 +0,0 @@ -import EventEmitter = require('events'); -import {Transform as TransformStream} from 'stream'; -import is from '@sindresorhus/is'; - -export function createProgressStream(name: 'downloadProgress' | 'uploadProgress', emitter: EventEmitter, totalBytes?: number | string): TransformStream { - let transformedBytes = 0; - - if (is.string(totalBytes)) { - totalBytes = Number(totalBytes); - } - - const progressStream = new TransformStream({ - transform(chunk, _encoding, callback) { - transformedBytes += chunk.length; - - const percent = totalBytes ? transformedBytes / (totalBytes as number) : 0; - - // Let `flush()` be responsible for emitting the last event - if (percent < 1) { - emitter.emit(name, { - percent, - transferred: transformedBytes, - total: totalBytes - }); - } - - callback(undefined, chunk); - }, - - flush(callback) { - emitter.emit(name, { - percent: 1, - transferred: transformedBytes, - total: totalBytes - }); - - callback(); - } - }); - - emitter.emit(name, { - percent: 0, - transferred: 0, - total: totalBytes - }); - - return progressStream; -} diff --git a/source/request-as-event-emitter.ts b/source/request-as-event-emitter.ts deleted file mode 100644 index d10725bf9..000000000 --- a/source/request-as-event-emitter.ts +++ /dev/null @@ -1,345 +0,0 @@ -import {ReadStream} from 'fs'; -import CacheableRequest = require('cacheable-request'); -import EventEmitter = require('events'); -import http = require('http'); -import stream = require('stream'); -import {URL} from 'url'; -import {promisify} from 'util'; -import is from '@sindresorhus/is'; -import timer, {ClientRequestWithTimings} from '@szmarczak/http-timer'; -import {ProxyStream} from './as-stream'; -import calculateRetryDelay from './calculate-retry-delay'; -import {CacheError, GotError, MaxRedirectsError, RequestError, TimeoutError} from './errors'; -import getResponse from './get-response'; -import {normalizeRequestArguments} from './normalize-arguments'; -import {createProgressStream} from './progress'; -import timedOut, {TimeoutError as TimedOutTimeoutError} from './utils/timed-out'; -import {GeneralError, NormalizedOptions, Response, requestSymbol} from './types'; -import urlToOptions from './utils/url-to-options'; -import pEvent = require('p-event'); - -const setImmediateAsync = async (): Promise => new Promise(resolve => setImmediate(resolve)); -const pipeline = promisify(stream.pipeline); - -const redirectCodes: ReadonlySet = new Set([300, 301, 302, 303, 304, 307, 308]); - -export interface RequestAsEventEmitter extends EventEmitter { - retry: (error: TimeoutError | RequestError) => boolean; - abort: () => void; -} - -export default (options: NormalizedOptions): RequestAsEventEmitter => { - const emitter = new EventEmitter() as RequestAsEventEmitter; - - const requestUrl = options.url.toString(); - const redirects: string[] = []; - let retryCount = 0; - - let currentRequest: http.ClientRequest; - - // `request.aborted` is a boolean since v11.0.0: https://github.com/nodejs/node/commit/4b00c4fafaa2ae8c41c1f78823c0feb810ae4723#diff-e3bc37430eb078ccbafe3aa3b570c91a - const isAborted = (): boolean => typeof currentRequest.aborted === 'number' || (currentRequest.aborted as unknown as boolean); - - const emitError = async (error: GeneralError): Promise => { - try { - for (const hook of options.hooks.beforeError) { - // eslint-disable-next-line no-await-in-loop - error = await hook(error); - } - - emitter.emit('error', error); - } catch (error_) { - emitter.emit('error', error_); - } - }; - - const get = async (): Promise => { - let httpOptions = await normalizeRequestArguments(options); - - const handleResponse = async (response: http.IncomingMessage): Promise => { - try { - /* istanbul ignore next: fixes https://github.com/electron/electron/blob/cbb460d47628a7a146adf4419ed48550a98b2923/lib/browser/api/net.js#L59-L65 */ - if (options.useElectronNet) { - response = new Proxy(response, { - get: (target, name) => { - if (name === 'trailers' || name === 'rawTrailers') { - return []; - } - - const value = (target as any)[name]; - return is.function_(value) ? value.bind(target) : value; - } - }); - } - - const typedResponse = response as Response; - const {statusCode} = typedResponse; - typedResponse.statusMessage = is.nonEmptyString(typedResponse.statusMessage) ? typedResponse.statusMessage : http.STATUS_CODES[statusCode]; - typedResponse.url = options.url.toString(); - typedResponse.requestUrl = requestUrl; - typedResponse.retryCount = retryCount; - typedResponse.redirectUrls = redirects; - typedResponse.request = {options}; - typedResponse.isFromCache = typedResponse.fromCache ?? false; - delete typedResponse.fromCache; - - if (!typedResponse.isFromCache) { - typedResponse.ip = response.socket.remoteAddress!; - } - - const rawCookies = typedResponse.headers['set-cookie']; - if (Reflect.has(options, 'cookieJar') && rawCookies) { - let promises: Array> = rawCookies.map(async (rawCookie: string) => options.cookieJar.setCookie(rawCookie, typedResponse.url)); - - if (options.ignoreInvalidCookies) { - promises = promises.map(async p => p.catch(() => {})); - } - - await Promise.all(promises); - } - - if (options.followRedirect && Reflect.has(typedResponse.headers, 'location') && redirectCodes.has(statusCode)) { - typedResponse.resume(); // We're being redirected, we don't care about the response. - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-boolean-literal-compare - if (statusCode === 303 || options.methodRewriting === false) { - if (options.method !== 'GET' && options.method !== 'HEAD') { - // Server responded with "see other", indicating that the resource exists at another location, - // and the client should request it from that location via GET or HEAD. - options.method = 'GET'; - } - - if (Reflect.has(options, 'body')) { - delete options.body; - } - - if (Reflect.has(options, 'json')) { - delete options.json; - } - - if (Reflect.has(options, 'form')) { - delete options.form; - } - } - - if (redirects.length >= options.maxRedirects) { - throw new MaxRedirectsError(typedResponse, options.maxRedirects, options); - } - - // Handles invalid URLs. See https://github.com/sindresorhus/got/issues/604 - const redirectBuffer = Buffer.from(typedResponse.headers.location, 'binary').toString(); - const redirectUrl = new URL(redirectBuffer, options.url); - - // Redirecting to a different site, clear cookies. - if (redirectUrl.hostname !== options.url.hostname && Reflect.has(options.headers, 'cookie')) { - delete options.headers.cookie; - } - - redirects.push(redirectUrl.toString()); - options.url = redirectUrl; - - for (const hook of options.hooks.beforeRedirect) { - // eslint-disable-next-line no-await-in-loop - await hook(options, typedResponse); - } - - emitter.emit('redirect', response, options); - - await get(); - return; - } - - await getResponse(typedResponse, options, emitter); - } catch (error) { - emitError(error); - } - }; - - const handleRequest = async (request: ClientRequestWithTimings): Promise => { - let isPiped = false; - let isFinished = false; - - // `request.finished` doesn't indicate whether this has been emitted or not - request.once('finish', () => { - isFinished = true; - }); - - currentRequest = request; - - const onError = (error: GeneralError): void => { - if (error instanceof TimedOutTimeoutError) { - error = new TimeoutError(error, request.timings!, options); - } else { - error = new RequestError(error, options); - } - - if (!emitter.retry(error as GotError)) { - emitError(error); - } - }; - - request.on('error', error => { - if (isPiped) { - // Check if it's caught by `stream.pipeline(...)` - if (!isFinished) { - return; - } - - // We need to let `TimedOutTimeoutError` through, because `stream.pipeline(…)` aborts the request automatically. - if (isAborted() && !(error instanceof TimedOutTimeoutError)) { - return; - } - } - - onError(error); - }); - - try { - timer(request); - timedOut(request, options.timeout, options.url); - - emitter.emit('request', request); - - const uploadStream = createProgressStream('uploadProgress', emitter, httpOptions.headers!['content-length'] as string); - - isPiped = true; - - await pipeline( - httpOptions.body!, - uploadStream, - request - ); - - request.emit('upload-complete'); - } catch (error) { - if (isAborted() && error.message === 'Premature close') { - // The request was aborted on purpose - return; - } - - onError(error); - } - }; - - if (options.cache) { - // `cacheable-request` doesn't support Node 10 API, fallback. - httpOptions = { - ...httpOptions, - ...urlToOptions(options.url) - }; - - // @ts-ignore `cacheable-request` has got invalid types - const cacheRequest = options.cacheableRequest!(httpOptions, handleResponse); - - cacheRequest.once('error', (error: GeneralError) => { - if (error instanceof CacheableRequest.RequestError) { - emitError(new RequestError(error, options)); - } else { - emitError(new CacheError(error, options)); - } - }); - - cacheRequest.once('request', handleRequest); - } else { - // Catches errors thrown by calling `requestFn(…)` - try { - handleRequest(httpOptions[requestSymbol](options.url, httpOptions, handleResponse)); - } catch (error) { - emitError(new RequestError(error, options)); - } - } - }; - - emitter.retry = error => { - let backoff: number; - - retryCount++; - - try { - backoff = options.retry.calculateDelay({ - attemptCount: retryCount, - retryOptions: options.retry, - error, - computedValue: calculateRetryDelay({ - attemptCount: retryCount, - retryOptions: options.retry, - error, - computedValue: 0 - }) - }); - } catch (error_) { - emitError(error_); - return false; - } - - if (backoff) { - const retry = async (options: NormalizedOptions): Promise => { - try { - for (const hook of options.hooks.beforeRetry) { - // eslint-disable-next-line no-await-in-loop - await hook(options, error, retryCount); - } - - await get(); - } catch (error_) { - emitError(error_); - } - }; - - setTimeout(retry, backoff, {...options, forceRefresh: true}); - return true; - } - - return false; - }; - - emitter.abort = () => { - emitter.prependListener('request', (request: http.ClientRequest) => { - request.abort(); - }); - - if (currentRequest) { - currentRequest.abort(); - } - }; - - (async () => { - try { - if (options.body instanceof ReadStream) { - await pEvent(options.body, 'open'); - } - - // Promises are executed immediately. - // If there were no `setImmediate` here, - // `promise.json()` would have no effect - // as the request would be sent already. - await setImmediateAsync(); - - for (const hook of options.hooks.beforeRequest) { - // eslint-disable-next-line no-await-in-loop - await hook(options); - } - - await get(); - } catch (error) { - emitError(error); - } - })(); - - return emitter; -}; - -export const proxyEvents = (proxy: EventEmitter | ProxyStream, emitter: RequestAsEventEmitter): void => { - const events = [ - 'request', - 'redirect', - 'uploadProgress', - 'downloadProgress' - ]; - - for (const event of events) { - emitter.on(event, (...args: unknown[]) => { - proxy.emit(event, ...args); - }); - } -}; diff --git a/source/types.ts b/source/types.ts deleted file mode 100644 index 4be4eccdf..000000000 --- a/source/types.ts +++ /dev/null @@ -1,274 +0,0 @@ -import http = require('http'); -import https = require('https'); -import Keyv = require('keyv'); -import CacheableRequest = require('cacheable-request'); -import PCancelable = require('p-cancelable'); -import ResponseLike = require('responselike'); -import {URL} from 'url'; -import {Readable as ReadableStream} from 'stream'; -import {Timings, IncomingMessageWithTimings} from '@szmarczak/http-timer'; -import CacheableLookup from 'cacheable-lookup'; -import {Except, Merge} from 'type-fest'; -import {GotReturn} from './create'; -import {GotError, HTTPError, MaxRedirectsError, ParseError, TimeoutError, RequestError} from './errors'; -import {Hooks} from './known-hook-events'; -import {URLOptions} from './utils/options-to-url'; - -export type GeneralError = Error | GotError | HTTPError | MaxRedirectsError | ParseError; - -export type Method = - | 'GET' - | 'POST' - | 'PUT' - | 'PATCH' - | 'HEAD' - | 'DELETE' - | 'OPTIONS' - | 'TRACE' - | 'get' - | 'post' - | 'put' - | 'patch' - | 'head' - | 'delete' - | 'options' - | 'trace'; - -export type ResponseType = 'json' | 'buffer' | 'text'; - -export interface Response extends IncomingMessageWithTimings { - body: BodyType; - statusCode: number; - - /** - The remote IP address. - - Note: Not available when the response is cached. This is hopefully a temporary limitation, see [lukechilds/cacheable-request#86](https://github.com/lukechilds/cacheable-request/issues/86). - */ - ip: string; - - fromCache?: boolean; - isFromCache?: boolean; - req?: http.ClientRequest; - requestUrl: string; - retryCount: number; - timings: Timings; - redirectUrls: string[]; - request: { - options: NormalizedOptions; - }; - url: string; -} - -// TODO: The `ResponseLike` type should be properly fixed instead: -// https://github.com/sindresorhus/got/pull/827/files#r323633794 -export interface ResponseObject extends Partial { - socket: { - remoteAddress: string; - }; -} - -export interface RetryObject { - attemptCount: number; - retryOptions: Required; - error: TimeoutError | RequestError; - computedValue: number; -} - -export type RetryFunction = (retryObject: RetryObject) => number; - -export type HandlerFunction = (options: NormalizedOptions, next: (options: NormalizedOptions) => T) => T | Promise; - -export interface DefaultRetryOptions { - limit: number; - methods: Method[]; - statusCodes: number[]; - errorCodes: string[]; - calculateDelay: RetryFunction; - maxRetryAfter?: number; -} - -export interface RetryOptions extends Partial { - retries?: number; -} - -export type RequestFunction = typeof http.request; - -export interface AgentByProtocol { - http?: http.Agent; - https?: https.Agent; -} - -export interface Delays { - lookup?: number; - connect?: number; - secureConnect?: number; - socket?: number; - response?: number; - send?: number; - request?: number; -} - -export type Headers = Record; - -interface ToughCookieJar { - getCookieString(currentUrl: string, options: {[key: string]: unknown}, cb: (err: Error | null, cookies: string) => void): void; - getCookieString(url: string, callback: (error: Error | null, cookieHeader: string) => void): void; - setCookie(cookieOrString: unknown, currentUrl: string, options: {[key: string]: unknown}, cb: (err: Error | null, cookie: unknown) => void): void; - setCookie(rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void): void; -} - -interface PromiseCookieJar { - getCookieString(url: string): Promise; - setCookie(rawCookie: string, url: string): Promise; -} - -export const requestSymbol = Symbol('request'); - -/* eslint-disable @typescript-eslint/indent */ -export type DefaultOptions = Merge< - Required< - Except< - GotOptions, - // Override - 'hooks' | - 'retry' | - 'timeout' | - 'context' | - '_pagination' | - - // Should not be present - 'agent' | - 'body' | - 'cookieJar' | - 'encoding' | - 'form' | - 'json' | - 'lookup' | - 'request' | - 'url' | - typeof requestSymbol - > - >, - { - hooks: Required; - retry: DefaultRetryOptions; - timeout: Delays; - context: {[key: string]: any}; - _pagination?: PaginationOptions['_pagination']; - } ->; -/* eslint-enable @typescript-eslint/indent */ - -export interface PaginationOptions { - _pagination?: { - transform?: (response: Response) => Promise | T[]; - filter?: (item: T, allItems: T[]) => boolean; - paginate?: (response: Response) => Options | false; - shouldContinue?: (item: T, allItems: T[]) => boolean; - countLimit?: number; - }; -} - -export interface GotOptions extends PaginationOptions { - [requestSymbol]?: RequestFunction; - url?: URL | string; - body?: string | Buffer | ReadableStream; - hooks?: Hooks; - decompress?: boolean; - isStream?: boolean; - encoding?: BufferEncoding; - method?: Method; - retry?: RetryOptions | number; - throwHttpErrors?: boolean; - cookieJar?: ToughCookieJar | PromiseCookieJar; - ignoreInvalidCookies?: boolean; - request?: RequestFunction; - agent?: http.Agent | https.Agent | boolean | AgentByProtocol; - cache?: string | CacheableRequest.StorageAdapter | false; - headers?: Headers; - responseType?: ResponseType; - resolveBodyOnly?: boolean; - followRedirect?: boolean; - prefixUrl?: URL | string; - timeout?: number | Delays; - dnsCache?: CacheableLookup | Map | Keyv | false; - useElectronNet?: boolean; - form?: {[key: string]: any}; - json?: {[key: string]: any}; - context?: {[key: string]: any}; - maxRedirects?: number; - lookup?: CacheableLookup['lookup']; - allowGetBody?: boolean; - methodRewriting?: boolean; -} - -export type Options = Merge>; - -export interface NormalizedOptions extends Options { - // Normalized Got options - headers: Headers; - hooks: Required; - timeout: Delays; - dnsCache: CacheableLookup | false; - lookup?: CacheableLookup['lookup']; - retry: Required; - prefixUrl: string; - method: Method; - url: URL; - cacheableRequest?: (options: string | URL | http.RequestOptions, callback?: (response: http.ServerResponse | ResponseLike) => void) => CacheableRequest.Emitter; - cookieJar?: PromiseCookieJar; - maxRedirects: number; - pagination?: Required['_pagination']>; - [requestSymbol]: RequestFunction; - - // Other values - decompress: boolean; - isStream: boolean; - throwHttpErrors: boolean; - ignoreInvalidCookies: boolean; - cache: CacheableRequest.StorageAdapter | false; - responseType: ResponseType; - resolveBodyOnly: boolean; - followRedirect: boolean; - useElectronNet: boolean; - methodRewriting: boolean; - allowGetBody: boolean; - context: {[key: string]: any}; - - // UNIX socket support - path?: string; -} - -export interface ExtendOptions extends Options { - handlers?: HandlerFunction[]; - mutableDefaults?: boolean; -} - -export interface Defaults { - options: DefaultOptions; - handlers: HandlerFunction[]; - mutableDefaults: boolean; - _rawHandlers?: HandlerFunction[]; -} - -export type URLOrOptions = Options | string; - -export interface Progress { - percent: number; - transferred: number; - total?: number; -} - -export interface GotEvents { - on(name: 'request', listener: (request: http.ClientRequest) => void): T; - on(name: 'response', listener: (response: Response) => void): T; - on(name: 'redirect', listener: (response: Response, nextOptions: NormalizedOptions) => void): T; - on(name: 'uploadProgress' | 'downloadProgress', listener: (progress: Progress) => void): T; -} - -export interface CancelableRequest extends PCancelable, GotEvents> { - json(): CancelableRequest; - buffer(): CancelableRequest; - text(): CancelableRequest; -} diff --git a/source/types/mimic-response/index.d.ts b/source/types/mimic-response/index.d.ts new file mode 100644 index 000000000..0fea28f8f --- /dev/null +++ b/source/types/mimic-response/index.d.ts @@ -0,0 +1,8 @@ +declare module 'mimic-response' { + import {IncomingMessage} from 'http'; + import {Transform as TransformStream} from 'stream'; + + declare function mimicResponse(input: IncomingMessage, output: TransformStream): void; + + export = mimicResponse; +} diff --git a/source/types/reflect/index.d.ts b/source/types/reflect/index.d.ts deleted file mode 100644 index c4625aabf..000000000 --- a/source/types/reflect/index.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -// The type-guarding behaviour is currently not supported as of TypeScript 3.7 -// https://github.com/microsoft/TypeScript/issues/30688 -declare namespace Reflect { - function has(target: T, propertyKey: Key): target is Required>; -} diff --git a/source/utils/dynamic-require.ts b/source/utils/dynamic-require.ts deleted file mode 100644 index 515f71995..000000000 --- a/source/utils/dynamic-require.ts +++ /dev/null @@ -1,3 +0,0 @@ -/* istanbul ignore file: used for webpack */ - -export default (moduleObject: NodeModule, moduleId: string): unknown => moduleObject.require(moduleId); diff --git a/source/utils/merge.ts b/source/utils/merge.ts deleted file mode 100644 index 9e6ccce67..000000000 --- a/source/utils/merge.ts +++ /dev/null @@ -1,32 +0,0 @@ -import {URL} from 'url'; -import is from '@sindresorhus/is'; -import {Merge} from 'type-fest'; - -export default function merge(target: Target, ...sources: Source[]): Merge { - for (const source of sources) { - for (const [key, sourceValue] of Object.entries(source)) { - const targetValue = target[key]; - - if (is.urlInstance(targetValue) && is.string(sourceValue)) { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = new URL(sourceValue, targetValue); - } else if (is.plainObject(sourceValue)) { - if (is.plainObject(targetValue)) { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = merge({}, targetValue, sourceValue); - } else { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = merge({}, sourceValue); - } - } else if (is.array(sourceValue)) { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = sourceValue.slice(); - } else { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = sourceValue; - } - } - } - - return target as Merge; -} diff --git a/source/utils/options-to-url.ts b/source/utils/options-to-url.ts deleted file mode 100644 index 777237b32..000000000 --- a/source/utils/options-to-url.ts +++ /dev/null @@ -1,113 +0,0 @@ -import {URL, URLSearchParams} from 'url'; - -function validateSearchParams(searchParams: Record): asserts searchParams is Record { - for (const value of Object.values(searchParams)) { - if (typeof value !== 'string' && typeof value !== 'number' && typeof value !== 'boolean' && value !== null) { - throw new TypeError(`The \`searchParams\` value '${String(value)}' must be a string, number, boolean or null`); - } - } -} - -export interface URLOptions { - href?: string; - origin?: string; - protocol?: string; - username?: string; - password?: string; - host?: string; - hostname?: string; - port?: string | number; - pathname?: string; - search?: string; - searchParams?: Record | URLSearchParams | string; - hash?: string; - - // The only accepted legacy URL options - path?: string; -} - -const keys: Array> = [ - 'protocol', - 'username', - 'password', - 'host', - 'hostname', - 'port', - 'pathname', - 'search', - 'hash' -]; - -export default (options: URLOptions): URL => { - let origin: string; - - if (options.path) { - if (options.pathname) { - throw new TypeError('Parameters `path` and `pathname` are mutually exclusive.'); - } - - if (options.search) { - throw new TypeError('Parameters `path` and `search` are mutually exclusive.'); - } - - if (options.searchParams) { - throw new TypeError('Parameters `path` and `searchParams` are mutually exclusive.'); - } - } - - if (Reflect.has(options, 'auth')) { - throw new TypeError('Parameter `auth` is deprecated. Use `username` / `password` instead.'); - } - - if (options.search && options.searchParams) { - throw new TypeError('Parameters `search` and `searchParams` are mutually exclusive.'); - } - - if (options.href) { - return new URL(options.href); - } - - if (options.origin) { - origin = options.origin; - } else { - if (!options.protocol) { - throw new TypeError('No URL protocol specified'); - } - - origin = `${options.protocol}//${options.hostname ?? options.host ?? ''}`; - } - - const url = new URL(origin); - - if (options.path) { - const searchIndex = options.path.indexOf('?'); - if (searchIndex === -1) { - options.pathname = options.path; - } else { - options.pathname = options.path.slice(0, searchIndex); - options.search = options.path.slice(searchIndex + 1); - } - } - - if (Reflect.has(options, 'path')) { - delete options.path; - } - - for (const key of keys) { - if (Reflect.has(options, key)) { - url[key] = options[key].toString(); - } - } - - if (options.searchParams) { - if (typeof options.searchParams !== 'string' && !(options.searchParams instanceof URLSearchParams)) { - validateSearchParams(options.searchParams); - } - - (new URLSearchParams(options.searchParams as Record)).forEach((value, key) => { - url.searchParams.append(key, value); - }); - } - - return url; -}; diff --git a/source/utils/proxy-events.ts b/source/utils/proxy-events.ts new file mode 100644 index 000000000..bd78b4a64 --- /dev/null +++ b/source/utils/proxy-events.ts @@ -0,0 +1,22 @@ +import {EventEmitter} from 'events'; + +type Fn = (...args: unknown[]) => void; +type Fns = {[key: string]: Fn}; + +export default function (from: EventEmitter, to: EventEmitter, events: string[]): () => void { + const fns: Fns = {}; + + for (const event of events) { + fns[event] = (...args: unknown[]) => { + to.emit(event, ...args); + }; + + from.on(event, fns[event]); + } + + return () => { + for (const event of events) { + from.off(event, fns[event]); + } + }; +} diff --git a/source/utils/supports-brotli.ts b/source/utils/supports-brotli.ts deleted file mode 100644 index 58b4ee283..000000000 --- a/source/utils/supports-brotli.ts +++ /dev/null @@ -1,3 +0,0 @@ -import zlib = require('zlib'); - -export default typeof zlib.createBrotliDecompress === 'function'; diff --git a/source/utils/timed-out.ts b/source/utils/timed-out.ts index 55b7b98f6..6ff1366b7 100644 --- a/source/utils/timed-out.ts +++ b/source/utils/timed-out.ts @@ -42,11 +42,18 @@ export class TimeoutError extends Error { } } -export default (request: ClientRequest, delays: Delays, options: TimedOutOptions): () => void => { - if (Reflect.has(request, reentry)) { +export default (request: ClientRequest, delaysOrNumber: Delays | number, options: TimedOutOptions): () => void => { + if (reentry in request) { return noop; } + let delays: Delays; + if (typeof delaysOrNumber === 'number') { + delays = {request: delaysOrNumber}; + } else { + delays = delaysOrNumber; + } + request[reentry] = true; const cancelers: Array = []; const {once, unhandleAll} = unhandler(); @@ -69,8 +76,7 @@ export default (request: ClientRequest, delays: Delays, options: TimedOutOptions const timeoutHandler = (delay: number, event: string): void => { if (request.socket) { - // @ts-ignore We do not want the `socket hang up` error - request.socket._hadError = true; + (request.socket as any)._hadError = true; } request.abort(); @@ -120,8 +126,7 @@ export default (request: ClientRequest, delays: Delays, options: TimedOutOptions } once(request, 'socket', (socket: net.Socket): void => { - // @ts-ignore Node typings doesn't have this property - const {socketPath} = request; + const {socketPath} = request as ClientRequest & {socketPath?: string}; /* istanbul ignore next: hard to test */ if (socket.connecting) { diff --git a/test/agent.ts b/test/agent.ts index 149588c34..61aab68cc 100644 --- a/test/agent.ts +++ b/test/agent.ts @@ -47,7 +47,9 @@ test('non-object agent option works with http', withServer, async (t, server, go t.truthy((await got({ rejectUnauthorized: false, - agent + agent: { + http: agent + } })).body); t.true(spy.calledOnce); @@ -64,7 +66,9 @@ test('non-object agent option works with https', withServer, async (t, server, g t.truthy((await got.secure({ rejectUnauthorized: false, - agent + agent: { + https: agent + } })).body); t.true(spy.calledOnce); @@ -126,7 +130,9 @@ test('socket connect listener cleaned up after request', withServer, async (t, s // eslint-disable-next-line no-await-in-loop await got.secure({ rejectUnauthorized: false, - agent + agent: { + https: agent + } }); } diff --git a/test/arguments.ts b/test/arguments.ts index 9ed09fd29..1f9dadf17 100644 --- a/test/arguments.ts +++ b/test/arguments.ts @@ -12,11 +12,18 @@ const echoUrl: Handler = (request, response) => { test('`url` is required', async t => { await t.throwsAsync( - // @ts-ignore Error tests got(''), { - instanceOf: TypeError, - message: 'No URL protocol specified' + message: 'Missing `url` property' + } + ); + + await t.throwsAsync( + got({ + url: '' + }), + { + message: 'Invalid URL: ' } ); }); @@ -33,8 +40,7 @@ test('`url` should be utf-8 encoded', async t => { test('throws if no arguments provided', async t => { // @ts-ignore Error tests await t.throwsAsync(got(), { - instanceOf: TypeError, - message: 'Missing `url` argument' + message: 'Missing `url` property' }); }); @@ -45,13 +51,7 @@ test('throws an error if the protocol is not specified', async t => { }); await t.throwsAsync(got({}), { - instanceOf: TypeError, - message: 'No URL protocol specified' - }); - - await t.throwsAsync(got({}), { - instanceOf: TypeError, - message: 'No URL protocol specified' + message: 'Missing `url` property' }); }); @@ -95,7 +95,7 @@ test('throws an error when legacy URL is passed', withServer, async (t, server, await t.throwsAsync( // @ts-ignore Error tests got(parse(`${server.url}/test`)), - {message: 'The legacy `url.Url` is deprecated. Use `URL` instead.'} + {message: 'The legacy `url.Url` has been deprecated. Use `URL` instead.'} ); }); @@ -107,20 +107,6 @@ test('overrides `searchParams` from options', withServer, async (t, server, got) { searchParams: { test: 'wow' - }, - cache: { - get(key: string) { - t.is(key, `cacheable-request:GET:${server.url}/?test=wow`); - }, - set(key: string) { - t.is(key, `cacheable-request:GET:${server.url}/?test=wow`); - }, - delete() { - return true; - }, - clear() { - return undefined; - } } } ); @@ -157,7 +143,6 @@ test('ignores empty searchParams object', withServer, async (t, server, got) => test('throws when passing body with a non payload method', async t => { // @ts-ignore Error tests await t.throwsAsync(got('https://example.com', {body: 'asdf'}), { - instanceOf: TypeError, message: 'The `GET` method cannot be used with a body' }); }); @@ -202,8 +187,7 @@ test('throws TypeError when `options.hooks` is not an object', async t => { // @ts-ignore Error tests got('https://example.com', {hooks: 'not object'}), { - instanceOf: TypeError, - message: 'Parameter `hooks` must be an Object, not string' + message: 'Expected value which is `predicate returns truthy for any value`, received value of type `Array`.' } ); }); @@ -213,8 +197,7 @@ test('throws TypeError when known `options.hooks` value is not an array', async // @ts-ignore Error tests got('https://example.com', {hooks: {beforeRequest: {}}}), { - instanceOf: TypeError, - message: 'Parameter `beforeRequest` must be an Array, not Object' + message: 'Parameter `beforeRequest` must be an Array, got Object' } ); }); @@ -225,7 +208,6 @@ test('throws TypeError when known `options.hooks` array item is not a function', // @ts-ignore Error tests got('https://example.com', {hooks: {beforeRequest: [{}]}}), { - instanceOf: TypeError, message: 'hook is not a function' } ); @@ -368,5 +350,62 @@ test('throws if `options.encoding` is `null`', async t => { test('`url` option and input argument are mutually exclusive', async t => { await t.throwsAsync(got('https://example.com', { url: 'https://example.com' - }), {message: 'The `url` option cannot be used if the input is a valid URL.'}); + }), {message: 'The `url` option is mutually exclusive with the `input` argument'}); +}); + +test('throws a helpful error when passing `followRedirects`', async t => { + await t.throwsAsync(got('https://example.com', { + // @ts-ignore For testing purposes + followRedirects: true + }), {message: 'The `followRedirects` option does not exist. Use `followRedirect` instead.'}); +}); + +test('merges `searchParams` instances', t => { + const instance = got.extend({ + searchParams: new URLSearchParams('a=1') + }, { + searchParams: new URLSearchParams('b=2') + }); + + t.is(instance.defaults.options.searchParams!.get('a'), '1'); + t.is(instance.defaults.options.searchParams!.get('b'), '2'); +}); + +test('throws a helpful error when passing `auth`', async t => { + await t.throwsAsync(got('https://example.com', { + // @ts-ignore For testing purposes + auth: 'username:password' + }), { + message: 'The legacy `url.Url` has been deprecated. Use `URL` instead.' + }); +}); + +test('throws when input starts with a slash and the `prefixUrl` option is present', async t => { + await t.throwsAsync(got('/asdf', {prefixUrl: 'https://example.com'}), { + message: '`input` must not start with a slash when using `prefixUrl`' + }); +}); + +test('throws on invalid `dnsCache` option', async t => { + // @ts-ignore Error tests + await t.throwsAsync(got('https://example.com', { + dnsCache: 123 + }), {message: 'Parameter `dnsCache` must be a CacheableLookup instance or a boolean, got number'}); +}); + +test('throws on invalid `agent` option', async t => { + await t.throwsAsync(got('https://example.com', { + agent: { + // @ts-ignore Error tests + asdf: 123 + } + }), {message: 'Expected the `options.agent` properties to be `http`, `https` or `http2`, got `asdf`'}); +}); + +test('fallbacks to native http if `request(...)` returns undefined', withServer, async (t, server, got) => { + server.get('/', echoUrl); + + const {body} = await got('', {request: () => undefined}); + + t.is(body, '/'); }); diff --git a/test/cache.ts b/test/cache.ts index 4e49cc75f..37f906dcd 100644 --- a/test/cache.ts +++ b/test/cache.ts @@ -4,6 +4,7 @@ import getStream = require('get-stream'); import {Handler} from 'express'; import {Response} from '../source'; import withServer from './helpers/with-server'; +import CacheableLookup from 'cacheable-lookup'; const cacheEndpoint: Handler = (_request, response) => { response.setHeader('Cache-Control', 'public, max-age=60'); @@ -81,11 +82,16 @@ test('redirects are cached and re-used internally', withServer, async (t, server server.get('/', cacheEndpoint); const cache = new Map(); - const firstResponse = await got('301', {cache}); - const secondResponse = await got('302', {cache}); + const A1 = await got('301', {cache}); + const B1 = await got('302', {cache}); + + const A2 = await got('301', {cache}); + const B2 = await got('302', {cache}); t.is(cache.size, 3); - t.is(firstResponse.body, secondResponse.body); + t.is(A1.body, B1.body); + t.is(A1.body, A2.body); + t.is(B1.body, B2.body); }); test('cached response has got options', withServer, async (t, server, got) => { @@ -135,10 +141,22 @@ test('doesn\'t cache response when received HTTP error', withServer, async (t, s }); test('DNS cache works', withServer, async (t, _server, got) => { - const map = new Map(); - await t.notThrowsAsync(got('https://example.com', {dnsCache: map, prefixUrl: ''})); + const instance = got.extend({ + dnsCache: true, + prefixUrl: '' + }); + + await t.notThrowsAsync(instance('https://example.com')); + + // @ts-ignore + t.is(instance.defaults.options.dnsCache!._cache.size, 1); +}); + +test('DNS cache works - CacheableLookup instance', withServer, async (t, _server, got) => { + const cache = new CacheableLookup(); + await t.notThrowsAsync(got('https://example.com', {dnsCache: cache, prefixUrl: ''})); - t.is(map.size, 1); + t.is((cache as any)._cache.size, 1); }); test('`isFromCache` stream property is undefined before the `response` event', withServer, async (t, server, got) => { diff --git a/test/cookies.ts b/test/cookies.ts index 68c5959e4..3f5abffb5 100644 --- a/test/cookies.ts +++ b/test/cookies.ts @@ -159,7 +159,7 @@ test('accepts custom `cookieJar` object', withServer, async (t, server, got) => async getCookieString(url: string) { t.is(typeof url, 'string'); - return cookies[url]; + return cookies[url] || ''; }, async setCookie(rawCookie: string, url: string) { diff --git a/test/create.ts b/test/create.ts index f0f7ad125..627fcdbda 100644 --- a/test/create.ts +++ b/test/create.ts @@ -96,15 +96,6 @@ test('extend keeps the old value if the new one is undefined', t => { ); }); -test('extend merges URL instances', t => { - // @ts-ignore Custom instance. - const a = got.extend({custom: new URL('https://example.com')}); - // @ts-ignore Custom instance. - const b = a.extend({custom: '/foo'}); - // @ts-ignore Custom instance. - t.is(b.defaults.options.custom.toString(), 'https://example.com/foo'); -}); - test('hooks are merged on got.extend()', t => { const hooksA = [() => {}]; const hooksB = [() => {}]; @@ -145,6 +136,12 @@ test('can set defaults to `got.mergeOptions(...)`', t => { }); t.true(instance.defaults.options.followRedirect); + + t.notThrows(() => { + instance.defaults.options = got.mergeOptions({}); + }); + + t.false(instance.defaults.options.followRedirect); }); test('can set mutable defaults using got.extend', t => { @@ -164,10 +161,15 @@ test('only plain objects are freezed', withServer, async (t, server, got) => { server.get('/', echoHeaders); const instance = got.extend({ - agent: new HttpAgent({keepAlive: true}) + agent: { + http: new HttpAgent({keepAlive: true}) + }, + mutableDefaults: true }); - await t.notThrowsAsync(() => instance('')); + t.notThrows(() => { + (instance.defaults.options.agent as any).http.keepAlive = true; + }); }); test('defaults are cloned on instance creation', t => { diff --git a/test/error.ts b/test/error.ts index f0ed7fc17..50107c43b 100644 --- a/test/error.ts +++ b/test/error.ts @@ -2,8 +2,7 @@ import {promisify} from 'util'; import http = require('http'); import stream = require('stream'); import test from 'ava'; -import proxyquire = require('proxyquire'); -import got, {GotError, HTTPError} from '../source'; +import got, {RequestError, HTTPError} from '../source'; import withServer from './helpers/with-server'; const pStreamPipeline = promisify(stream.pipeline); @@ -34,7 +33,7 @@ test('properties', withServer, async (t, server, got) => { }); test('catches dns errors', async t => { - const error = await t.throwsAsync(got('http://doesntexist', {retry: 0})); + const error = await t.throwsAsync(got('http://doesntexist', {retry: 0})); t.truthy(error); t.regex(error.message, /getaddrinfo ENOTFOUND/); t.is(error.options.url.host, 'doesntexist'); @@ -102,11 +101,11 @@ test('contains Got options', withServer, async (t, server, got) => { response.end(); }); - const options = { + const options: {agent: false} = { agent: false }; - const error = await t.throwsAsync(got(options)); + const error = await t.throwsAsync(got(options)); t.is(error.options.agent, options.agent); }); @@ -140,10 +139,23 @@ test('`http.request` pipe error', async t => { // @ts-ignore Error tests request: () => { const proxy = new stream.PassThrough(); + + const anyProxy = proxy as any; + anyProxy.socket = { + remoteAddress: '', + prependOnceListener: () => {} + }; + + anyProxy.headers = {}; + + anyProxy.abort = () => {}; + proxy.resume(); - proxy.once('pipe', () => { + proxy.read = () => { proxy.destroy(new Error(message)); - }); + + return null; + }; return proxy; }, @@ -166,30 +178,12 @@ test('`http.request` error through CacheableRequest', async t => { }); }); -test('catches error in mimicResponse', withServer, async (t, server) => { - server.get('/', (_request, response) => { - response.end('ok'); - }); - - const mimicResponse = (): never => { - throw new Error('Error in mimic-response'); - }; - - mimicResponse['@global'] = true; - - const proxiedGot = proxyquire('../source', { - 'mimic-response': mimicResponse - }); - - await t.throwsAsync(proxiedGot(server.url), {message: 'Error in mimic-response'}); -}); - test('errors are thrown directly when options.stream is true', t => { t.throws(() => { // @ts-ignore Error tests got('https://example.com', {isStream: true, hooks: false}); }, { - message: 'Parameter `hooks` must be an Object, not boolean' + message: 'Expected value which is `predicate returns truthy for any value`, received value of type `Array`.' }); }); diff --git a/test/gzip.ts b/test/gzip.ts index c0d15ec57..2b14e35e2 100644 --- a/test/gzip.ts +++ b/test/gzip.ts @@ -72,7 +72,7 @@ test('decompress option opts out of decompressing', withServer, async (t, server response.end(gzipData); }); - const {body} = await got({decompress: false}); + const {body} = await got({decompress: false, responseType: 'buffer'}); t.is(Buffer.compare(body, gzipData), 0); }); diff --git a/test/headers.ts b/test/headers.ts index e47df3afe..da1d31ade 100644 --- a/test/headers.ts +++ b/test/headers.ts @@ -5,9 +5,10 @@ import test from 'ava'; import {Handler} from 'express'; import FormData = require('form-data'); import got, {Headers} from '../source'; -import supportsBrotli from '../source/utils/supports-brotli'; import withServer from './helpers/with-server'; +const supportsBrotli = typeof (process.versions as any).brotli === 'string'; + const echoHeaders: Handler = (request, response) => { request.resume(); response.end(JSON.stringify(request.headers)); @@ -42,10 +43,8 @@ test('does not remove user headers from `url` object argument', withServer, asyn server.get('/', echoHeaders); const headers = (await got({ - hostname: server.hostname, - port: server.port, + url: `http://${server.hostname}:${server.port}`, responseType: 'json', - protocol: 'http:', headers: { 'X-Request-Id': 'value' } @@ -123,6 +122,21 @@ test('sets `content-length` to `0` when requesting PUT with empty body', withSer t.is(headers['content-length'], '0'); }); +test('form manual `content-type` header', withServer, async (t, server, got) => { + server.post('/', echoHeaders); + + const {body} = await got.post({ + headers: { + 'content-type': 'custom' + }, + form: { + a: 1 + } + }); + const headers = JSON.parse(body); + t.is(headers['content-type'], 'custom'); +}); + test('form-data manual `content-type` header', withServer, async (t, server, got) => { server.post('/', echoHeaders); @@ -184,13 +198,13 @@ test('buffer as `options.body` sets `content-length` header', withServer, async }); test('throws on null value headers', async t => { - // @ts-ignore Error tests await t.throwsAsync(got({ + url: 'https://example.com', + // @ts-ignore Testing purposes headers: { 'user-agent': null } }), { - instanceOf: TypeError, message: 'Use `undefined` instead of `null` to delete the `user-agent` header' }); }); diff --git a/test/helpers/with-server.ts b/test/helpers/with-server.ts index 3c732d86e..c4aa3872c 100644 --- a/test/helpers/with-server.ts +++ b/test/helpers/with-server.ts @@ -4,7 +4,7 @@ import http = require('http'); import tempy = require('tempy'); import createTestServer = require('create-test-server'); import lolex = require('lolex'); -import got, {Defaults} from '../../source'; +import got, {InstanceDefaults} from '../../source'; import {ExtendedGot, ExtendedHttpServer, ExtendedTestServer, GlobalClock, InstalledClock} from './types'; export type RunTestWithServer = (t: test.ExecutionContext, server: ExtendedTestServer, got: ExtendedGot, clock: GlobalClock) => Promise | void; @@ -19,7 +19,7 @@ const generateHook = ({install}: {install?: boolean}): test.Macro<[RunTestWithSe } }) as ExtendedTestServer; - const options: Defaults = { + const options: InstanceDefaults = { // @ts-ignore Augmenting for test detection avaTest: t.title, handlers: [ diff --git a/test/hooks.ts b/test/hooks.ts index 8faa68109..76e2f7f62 100644 --- a/test/hooks.ts +++ b/test/hooks.ts @@ -1,8 +1,10 @@ +import {URL} from 'url'; import test from 'ava'; import getStream from 'get-stream'; import delay = require('delay'); import {Handler} from 'express'; -import got from '../source'; +import Responselike = require('responselike'); +import got, {RequestError} from '../source'; import withServer from './helpers/with-server'; const errorString = 'oops'; @@ -12,6 +14,10 @@ const echoHeaders: Handler = (request, response) => { response.end(JSON.stringify(request.headers)); }; +const echoUrl: Handler = (request, response) => { + response.end(request.url); +}; + const retryEndpoint: Handler = (request, response) => { if (request.headers.foo) { response.statusCode = 302; @@ -104,12 +110,16 @@ test('catches afterResponse thrown errors', withServer, async (t, server, got) = }), {message: errorString}); }); -test('throws a helpful error when passing async function as init hook', async t => { - await t.throwsAsync(got('https://example.com', { +test('accepts an async function as init hook', async t => { + await got('https://example.com', { hooks: { - init: [async () => {}] + init: [ + async () => { + t.pass(); + } + ] } - }), {message: 'The `init` hook must be a synchronous function'}); + }); }); test('catches beforeRequest promise rejections', async t => { @@ -192,7 +202,6 @@ test('init is called with options', withServer, async (t, server, got) => { hooks: { init: [ options => { - t.is(options.url, undefined); t.is(options.context, context); } ] @@ -210,7 +219,6 @@ test('init from defaults is called with options', withServer, async (t, server, hooks: { init: [ options => { - t.is(options.url, undefined); t.is(options.context, context); } ] @@ -271,6 +279,29 @@ test('beforeRequest allows modifications', withServer, async (t, server, got) => t.is(body.foo, 'bar'); }); +test('returning HTTP response from a beforeRequest hook', withServer, async (t, server, got) => { + server.get('/', echoUrl); + + const {statusCode, headers, body} = await got({ + hooks: { + beforeRequest: [ + () => { + return new Responselike( + 200, + {foo: 'bar'}, + Buffer.from('Hi!'), + '' + ); + } + ] + } + }); + + t.is(statusCode, 200); + t.is(headers.foo, 'bar'); + t.is(body, 'Hi!'); +}); + test('beforeRedirect is called with options and response', withServer, async (t, server, got) => { server.get('/', echoHeaders); server.get('/redirect', redirectEndpoint); @@ -579,7 +610,6 @@ test('throwing in a beforeError hook - promise', withServer, async (t, server, g response.end('ok'); }); - // @ts-ignore Error tests await t.throwsAsync(got({ hooks: { afterResponse: [ @@ -588,11 +618,11 @@ test('throwing in a beforeError hook - promise', withServer, async (t, server, g } ], beforeError: [ - () => { + (): never => { throw new Error('foobar'); }, () => { - t.fail('This shouldn\'t be called at all'); + throw new Error('This shouldn\'t be called at all'); } ] } @@ -600,7 +630,6 @@ test('throwing in a beforeError hook - promise', withServer, async (t, server, g }); test('throwing in a beforeError hook - stream', withServer, async (t, _server, got) => { - // @ts-ignore Error tests await t.throwsAsync(getStream(got.stream({ hooks: { beforeError: [ @@ -608,7 +637,7 @@ test('throwing in a beforeError hook - stream', withServer, async (t, _server, g throw new Error('foobar'); }, () => { - t.fail('This shouldn\'t be called at all'); + throw new Error('This shouldn\'t be called at all'); } ] } @@ -654,9 +683,13 @@ test('beforeError allows modifications', async t => { throw error; }, hooks: { - beforeError: [() => { - return new Error(errorString2); - }] + beforeError: [ + error => { + const newError = new Error(errorString2); + + return new RequestError(newError.message, newError, error.options); + } + ] } }), {message: errorString2}); }); @@ -669,9 +702,9 @@ test('does not break on `afterResponse` hook with JSON mode', withServer, async afterResponse: [ (response, retryWithMergedOptions) => { if (response.statusCode === 404) { - return retryWithMergedOptions({ - path: '/foobar' - }); + const url = new URL('/foobar', response.url); + + return retryWithMergedOptions({url}); } return response; @@ -688,7 +721,7 @@ test('catches HTTPErrors', withServer, async (t, _server, got) => { await t.throwsAsync(got({ hooks: { beforeError: [ - (error: Error) => { + error => { t.true(error instanceof got.HTTPError); return error; } diff --git a/test/http.ts b/test/http.ts index 6ca819ab0..8630bfe7d 100644 --- a/test/http.ts +++ b/test/http.ts @@ -92,15 +92,6 @@ test('`searchParams` option', withServer, async (t, server, got) => { t.is((await got({searchParams: 'recent=true'})).body, 'recent'); }); -test('response has `requestUrl` property even if `url` is an object', withServer, async (t, server, got) => { - server.get('/', (_request, response) => { - response.end('ok'); - }); - - t.is((await got({hostname: server.hostname, port: server.port})).requestUrl, `${server.url}/`); - t.is((await got({hostname: server.hostname, port: server.port, protocol: 'http:'})).requestUrl, `${server.url}/`); -}); - test('response contains url', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.end('ok'); @@ -114,11 +105,39 @@ test('response contains got options', withServer, async (t, server, got) => { response.end('ok'); }); - const options = { - username: 'foo' - }; + { + const options = { + username: 'foo', + password: 'bar' + }; + + const {options: normalizedOptions} = (await got(options)).request; + + t.is(normalizedOptions.username, options.username); + t.is(normalizedOptions.password, options.password); + } + + { + const options = { + username: 'foo' + }; + + const {options: normalizedOptions} = (await got(options)).request; + + t.is(normalizedOptions.username, options.username); + t.is(normalizedOptions.password, ''); + } + + { + const options = { + password: 'bar' + }; + + const {options: normalizedOptions} = (await got(options)).request; - t.is((await got(options)).request.options.username, options.username); + t.is(normalizedOptions.username, ''); + t.is(normalizedOptions.password, options.password); + } }); test('socket destroyed by the server throws ECONNRESET', withServer, async (t, server, got) => { diff --git a/test/https.ts b/test/https.ts index 8b7ff6419..c4e4d79ac 100644 --- a/test/https.ts +++ b/test/https.ts @@ -1,4 +1,5 @@ import test from 'ava'; +import got from '../source'; import withServer from './helpers/with-server'; test('https request without ca', withServer, async (t, server, got) => { @@ -20,3 +21,16 @@ test('https request with ca', withServer, async (t, server, got) => { }); t.is(body, 'ok'); }); + +test('http2', async t => { + const promise = got('https://httpbin.org/anything', { + http2: true + }); + + const {headers, body} = await promise; + await promise.json(); + + // @ts-ignore Pseudo headers may not be strings + t.is(headers[':status'], 200); + t.is(typeof body, 'string'); +}); diff --git a/test/options-to-url.ts b/test/options-to-url.ts deleted file mode 100644 index 1434de9db..000000000 --- a/test/options-to-url.ts +++ /dev/null @@ -1,150 +0,0 @@ -import test from 'ava'; -import is from '@sindresorhus/is'; -import optionsToUrl from '../source/utils/options-to-url'; - -const origin = 'https://google.com'; - -test('`path` and `pathname` are mutually exclusive', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({path: 'a', pathname: 'a'}); - }, { - message: 'Parameters `path` and `pathname` are mutually exclusive.' - }); -}); - -test('`path` and `search` are mutually exclusive', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({path: 'a', search: 'a'}); - }, { - message: 'Parameters `path` and `search` are mutually exclusive.' - }); -}); - -test('`path` and `searchParams` are mutually exclusive', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({path: 'a', searchParams: {}}); - }, { - message: 'Parameters `path` and `searchParams` are mutually exclusive.' - }); -}); - -test('`path` option', t => { - { - const url = optionsToUrl({origin, path: '/x?a=1'}); - t.is(url.href, `${origin}/x?a=1`); - t.true(is.urlInstance(url)); - } - - { - const url = optionsToUrl({origin, path: '/foobar'}); - t.is(url.href, `${origin}/foobar`); - t.true(is.urlInstance(url)); - } -}); - -test('`auth` is deprecated', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({auth: ''}); - }, { - message: 'Parameter `auth` is deprecated. Use `username` / `password` instead.' - }); -}); - -test('`search` and `searchParams` are mutually exclusive', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({search: 'a', searchParams: {}}); - }, { - message: 'Parameters `search` and `searchParams` are mutually exclusive.' - }); -}); - -test('`href` option', t => { - const url = optionsToUrl({href: origin}); - t.is(url.href, `${origin}/`); - t.true(is.urlInstance(url)); -}); - -test('`origin` option', t => { - const url = optionsToUrl({origin}); - t.is(url.href, `${origin}/`); - t.true(is.urlInstance(url)); -}); - -test('throws if no protocol specified', t => { - t.throws(() => { - optionsToUrl({}); - }, { - message: 'No URL protocol specified' - }); -}); - -test('`port` option', t => { - const url = optionsToUrl({origin, port: 8888}); - t.is(url.href, `${origin}:8888/`); - t.true(is.urlInstance(url)); -}); - -test('`protocol` option', t => { - const url = optionsToUrl({origin, protocol: 'http:'}); - t.is(url.href, 'http://google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`username` option', t => { - const url = optionsToUrl({origin, username: 'username'}); - t.is(url.href, 'https://username@google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`password` option', t => { - const url = optionsToUrl({origin, password: 'password'}); - t.is(url.href, 'https://:password@google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`username` option combined with `password` option', t => { - const url = optionsToUrl({origin, username: 'username', password: 'password'}); - t.is(url.href, 'https://username:password@google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`host` option', t => { - const url = optionsToUrl({protocol: 'https:', host: 'google.com'}); - t.is(url.href, 'https://google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`hostname` option', t => { - const url = optionsToUrl({protocol: 'https:', hostname: 'google.com'}); - t.is(url.href, 'https://google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`pathname` option', t => { - const url = optionsToUrl({protocol: 'https:', hostname: 'google.com', pathname: '/foobar'}); - t.is(url.href, 'https://google.com/foobar'); - t.true(is.urlInstance(url)); -}); - -test('`search` option', t => { - const url = optionsToUrl({protocol: 'https:', hostname: 'google.com', search: '?a=1'}); - t.is(url.href, 'https://google.com/?a=1'); - t.true(is.urlInstance(url)); -}); - -test('`hash` option', t => { - const url = optionsToUrl({protocol: 'https:', hostname: 'google.com', hash: 'foobar'}); - t.is(url.href, 'https://google.com/#foobar'); - t.true(is.urlInstance(url)); -}); - -test('merges provided `searchParams`', t => { - const url = optionsToUrl({origin: 'https://google.com/?a=1', searchParams: {b: 2}}); - t.is(url.href, 'https://google.com/?a=1&b=2'); - t.true(is.urlInstance(url)); -}); diff --git a/test/pagination.ts b/test/pagination.ts index 031294dbe..e4cc197e3 100644 --- a/test/pagination.ts +++ b/test/pagination.ts @@ -1,3 +1,4 @@ +import {URL} from 'url'; import test from 'ava'; import got, {Response} from '../source'; import withServer from './helpers/with-server'; @@ -72,7 +73,7 @@ test('filters elements', withServer, async (t, server, got) => { const result = await got.paginate.all({ _pagination: { - filter: element => element !== 2 + filter: (element: unknown) => element !== 2 } }); @@ -109,11 +110,15 @@ test('custom paginate function', withServer, async (t, server, got) => { const result = await got.paginate.all({ _pagination: { paginate: response => { - if (response.request.options.path === '/?page=3') { + const url = new URL(response.url); + + if (url.search === '?page=3') { return false; } - return {path: '/?page=3'}; + url.search = '?page=3'; + + return {url}; } } }); @@ -211,7 +216,8 @@ test('throws if the `pagination` option does not have `filter` property', async _pagination: { ...resetPagination, transform: thrower, - shouldContinue: thrower + shouldContinue: thrower, + paginate: thrower }, prefixUrl: 'https://example.com' }); diff --git a/test/post.ts b/test/post.ts index db3691c05..1aa6cb5d5 100644 --- a/test/post.ts +++ b/test/post.ts @@ -6,7 +6,7 @@ import delay = require('delay'); import {Handler} from 'express'; import getStream = require('get-stream'); import toReadableStream = require('to-readable-stream'); -import got from '../source'; +import got, {UploadError} from '../source'; import withServer from './helpers/with-server'; const pStreamPipeline = promisify(stream.pipeline); @@ -37,7 +37,6 @@ test('invalid body', async t => { // @ts-ignore Error tests got.post('https://example.com', {body: {}}), { - instanceOf: TypeError, message: 'The `body` option must be a stream.Readable, string or Buffer' } ); @@ -80,7 +79,6 @@ test('does NOT support sending arrays as forms', withServer, async (t, server, g await t.throwsAsync(got.post({ form: ['such', 'wow'] }), { - instanceOf: TypeError, message: 'Each query pair must be an iterable [name, value] tuple' }); }); @@ -213,7 +211,6 @@ test('`content-type` header is not overriden when object in `options.body`', wit test('throws when form body is not a plain object or array', async t => { // @ts-ignore Manual test await t.throwsAsync(got.post('https://example.com', {form: 'such=wow'}), { - instanceOf: TypeError, message: 'The `form` option must be an Object' }); }); @@ -306,3 +303,26 @@ test('catches body errors before calling pipeline() - stream', withServer, async // Wait for unhandled errors await delay(100); }); + +test('throws on upload error', withServer, async (t, server, got) => { + server.post('/', defaultEndpoint); + + const body = new stream.PassThrough(); + const message = 'oh no'; + + await t.throwsAsync(getStream(got.stream.post({ + body, + hooks: { + beforeRequest: [ + () => { + process.nextTick(() => { + body.destroy(new Error(message)); + }); + } + ] + } + })), { + instanceOf: UploadError, + message + }); +}); diff --git a/test/progress.ts b/test/progress.ts index 8088bf205..912f7cb35 100644 --- a/test/progress.ts +++ b/test/progress.ts @@ -22,17 +22,21 @@ const checkEvents: Macro<[Progress[], number?]> = (t, events, bodySize = undefin } for (const [index, event] of events.entries()) { + const isLastEvent = index === events.length - 1; + if (is.number(bodySize)) { t.is(event.percent, event.transferred / bodySize); t.true(event.percent > lastEvent.percent); + t.true(event.transferred > lastEvent.transferred); + } else if (isLastEvent) { + t.is(event.percent, 1); + t.is(event.transferred, lastEvent.transferred); + t.is(event.total, event.transferred); } else { - const isLastEvent = index === events.length - 1; - t.is(event.percent, isLastEvent ? 1 : 0); + t.is(event.percent, 0); + t.true(event.transferred > lastEvent.transferred); } - t.true(event.transferred >= lastEvent.transferred); - t.is(event.total, bodySize); - lastEvent = event; } }; @@ -199,7 +203,7 @@ test('upload progress - no body', withServer, async (t, server, got) => { { percent: 0, transferred: 0, - total: 0 + total: undefined }, { percent: 1, diff --git a/test/promise.ts b/test/promise.ts index 2e4054ce6..8e738ca0d 100644 --- a/test/promise.ts +++ b/test/promise.ts @@ -1,5 +1,4 @@ -import {ClientRequest} from 'http'; -import {Transform as TransformStream} from 'stream'; +import {ClientRequest, IncomingMessage} from 'http'; import test from 'ava'; import {Response} from '../source'; import withServer from './helpers/with-server'; @@ -22,7 +21,7 @@ test('emits response event as promise', withServer, async (t, server, got) => { }); await got('').json().on('response', (response: Response) => { - t.true(response instanceof TransformStream); + t.true(response instanceof IncomingMessage); t.true(response.readable); t.is(response.statusCode, 200); t.is(response.ip, '127.0.0.1'); diff --git a/test/redirects.ts b/test/redirects.ts index 5b74c7689..a9b8ac4ac 100644 --- a/test/redirects.ts +++ b/test/redirects.ts @@ -115,16 +115,6 @@ test('searchParams are not breaking redirects', withServer, async (t, server, go t.is((await got('relativeSearchParam', {searchParams: 'bang=1'})).body, 'reached'); }); -test('hostname + pathname are not breaking redirects', withServer, async (t, server, got) => { - server.get('/', reachedHandler); - server.get('/relative', relativeHandler); - - t.is((await got('relative', { - hostname: server.hostname, - pathname: '/relative' - })).body, 'reached'); -}); - test('redirects GET and HEAD requests', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.writeHead(308, { @@ -282,7 +272,7 @@ test('throws on malformed redirect URI', withServer, async (t, server, got) => { }); await t.throwsAsync(got(''), { - name: 'URIError' + message: 'URI malformed' }); }); @@ -385,7 +375,7 @@ test('body is passed on POST redirect', withServer, async (t, server, got) => { t.is(body, 'foobar'); }); -test('method overwriting can be turned off', withServer, async (t, server, got) => { +test('method rewriting can be turned off', withServer, async (t, server, got) => { server.post('/redirect', (_request, response) => { response.writeHead(302, { location: '/' diff --git a/test/response-parse.ts b/test/response-parse.ts index 7ea43eedf..5184ba078 100644 --- a/test/response-parse.ts +++ b/test/response-parse.ts @@ -101,7 +101,7 @@ test('parses non-200 responses', withServer, async (t, server, got) => { response.end(jsonResponse); }); - const error = await t.throwsAsync(got({responseType: 'json'}), {instanceOf: HTTPError}); + const error = await t.throwsAsync(got({responseType: 'json', retry: 0}), {instanceOf: HTTPError}); t.deepEqual(error.response.body, dog); }); @@ -111,7 +111,7 @@ test('ignores errors on invalid non-200 responses', withServer, async (t, server response.end('Internal error'); }); - const error = await t.throwsAsync(got({responseType: 'json'}), { + const error = await t.throwsAsync(got({responseType: 'json', retry: 0}), { instanceOf: got.HTTPError, message: 'Response code 500 (Internal Server Error)' }); diff --git a/test/retry.ts b/test/retry.ts index a48a22a39..b3a22fa29 100644 --- a/test/retry.ts +++ b/test/retry.ts @@ -22,7 +22,7 @@ const createSocketTimeoutStream = (): http.ClientRequest => { const stream = new PassThroughStream(); // @ts-ignore Mocking the behaviour of a ClientRequest stream.setTimeout = (ms, callback) => { - callback(); + process.nextTick(callback); }; // @ts-ignore Mocking the behaviour of a ClientRequest @@ -132,6 +132,7 @@ test('custom error codes', async t => { const error = await t.throwsAsync(got('https://example.com', { request: () => { const emitter = new EventEmitter() as http.ClientRequest; + emitter.abort = () => {}; emitter.end = () => {}; const error = new Error('Snap!'); @@ -204,7 +205,7 @@ test('doesn\'t retry on 413 with empty statusCodes and methods', withServer, asy const {statusCode, retryCount} = await got({ throwHttpErrors: false, retry: { - retries: 1, + limit: 1, statusCodes: [], methods: [] } @@ -219,7 +220,7 @@ test('doesn\'t retry on 413 with empty methods', withServer, async (t, server, g const {statusCode, retryCount} = await got({ throwHttpErrors: false, retry: { - retries: 1, + limit: 1, statusCodes: [413], methods: [] } diff --git a/test/stream.ts b/test/stream.ts index 18e0ded0a..1eaaa3926 100644 --- a/test/stream.ts +++ b/test/stream.ts @@ -68,23 +68,21 @@ test('returns writeable stream', withServer, async (t, server, got) => { test('throws on write if body is specified', withServer, (t, server, got) => { server.post('/', postHandler); - t.throws(() => { - got.stream.post({body: 'wow'}).end('wow'); - }, { - message: 'Got\'s stream is not writable when the `body`, `json` or `form` option is used' - }); + const streams = [ + got.stream.post({body: 'wow'}), + got.stream.post({json: {}}), + got.stream.post({form: {}}) + ]; - t.throws(() => { - got.stream.post({json: {}}).end('wow'); - }, { - message: 'Got\'s stream is not writable when the `body`, `json` or `form` option is used' - }); + for (const stream of streams) { + t.throws(() => { + stream.end('wow'); + }, { + message: 'The payload has been already provided' + }); - t.throws(() => { - got.stream.post({form: {}}).end('wow'); - }, { - message: 'Got\'s stream is not writable when the `body`, `json` or `form` option is used' - }); + stream.destroy(); + } }); test('does not throw if using stream and passing a json option', withServer, async (t, server, got) => { @@ -102,11 +100,15 @@ test('does not throw if using stream and passing a form option', withServer, asy test('throws on write if no payload method is present', withServer, (t, server, got) => { server.post('/', postHandler); + const stream = got.stream.get(''); + t.throws(() => { - got.stream.get('').end('wow'); + stream.end('wow'); }, { - message: 'The `GET` method cannot be used with a body' + message: 'The payload has been already provided' }); + + stream.destroy(); }); test('has request event', withServer, async (t, server, got) => { diff --git a/test/timeout.ts b/test/timeout.ts index b16e49f3c..a0c709250 100644 --- a/test/timeout.ts +++ b/test/timeout.ts @@ -87,7 +87,7 @@ test.serial('socket timeout', async t => { const stream = new PassThroughStream(); // @ts-ignore Mocking the behaviour of a ClientRequest stream.setTimeout = (ms, callback) => { - callback(); + process.nextTick(callback); }; // @ts-ignore Mocking the behaviour of a ClientRequest @@ -133,11 +133,13 @@ test.serial('send timeout (keepalive)', withServerAndLolex, async (t, server, go response.end('ok'); }); - await got('prime', {agent: keepAliveAgent}); + await got('prime', {agent: {http: keepAliveAgent}}); await t.throwsAsync( got.post({ - agent: keepAliveAgent, + agent: { + http: keepAliveAgent + }, timeout: {send: 1}, retry: 0, body: slowDataStream(clock) @@ -198,10 +200,12 @@ test.serial('response timeout (keepalive)', withServerAndLolex, async (t, server response.end('ok'); }); - await got('prime', {agent: keepAliveAgent}); + await got('prime', {agent: {http: keepAliveAgent}}); const request = got({ - agent: keepAliveAgent, + agent: { + http: keepAliveAgent + }, timeout: {response: 1}, retry: 0 }).on('request', (request: http.ClientRequest) => { @@ -248,7 +252,8 @@ test.serial('connect timeout', withServerAndLolex, async (t, _server, got, clock test.serial('connect timeout (ip address)', withServerAndLolex, async (t, _server, got, clock) => { await t.throwsAsync( got({ - hostname: '127.0.0.1', + url: 'http://127.0.0.1', + prefixUrl: '', createConnection: options => { const socket = new net.Socket(options as object as net.SocketConstructorOpts); // @ts-ignore We know that it is readonly, but we have to test it @@ -336,7 +341,8 @@ test.serial('lookup timeout no error (ip address)', withServerAndLolex, async (t server.get('/', defaultHandler(clock)); await t.notThrowsAsync(got({ - hostname: '127.0.0.1', + url: `http://127.0.0.1:${server.port}`, + prefixUrl: '', timeout: {lookup: 1}, retry: 0 })); @@ -348,9 +354,9 @@ test.serial('lookup timeout no error (keepalive)', withServerAndLolex, async (t, response.end('ok'); }); - await got('prime', {agent: keepAliveAgent}); + await got('prime', {agent: {http: keepAliveAgent}}); await t.notThrowsAsync(got({ - agent: keepAliveAgent, + agent: {http: keepAliveAgent}, timeout: {lookup: 1}, retry: 0 }).on('request', (request: http.ClientRequest) => { @@ -358,10 +364,12 @@ test.serial('lookup timeout no error (keepalive)', withServerAndLolex, async (t, t.fail('connect event fired, invalidating test'); }); })); + + keepAliveAgent.destroy(); }); -test.serial('retries on timeout', withServerAndLolex, async (t, server, got, clock) => { - server.get('/', defaultHandler(clock)); +test.serial('retries on timeout', withServer, async (t, server, got) => { + server.get('/', () => {}); let hasTried = false; await t.throwsAsync(got({ @@ -516,7 +524,7 @@ test.serial('no memory leak when using socket timeout and keepalive agent', with server.get('/', defaultHandler(clock)); const promise = got({ - agent: keepAliveAgent, + agent: {http: keepAliveAgent}, timeout: {socket: requestDelay * 2} }); @@ -530,6 +538,8 @@ test.serial('no memory leak when using socket timeout and keepalive agent', with await promise; t.is(socket.listenerCount('timeout'), 0); + + keepAliveAgent.destroy(); }); test('ensure there are no new timeouts after cancelation', t => { diff --git a/test/unix-socket.ts b/test/unix-socket.ts index 8fca64df5..79d583e7b 100644 --- a/test/unix-socket.ts +++ b/test/unix-socket.ts @@ -31,7 +31,7 @@ if (process.platform !== 'win32') { }); test('throws on invalid URL', async t => { - await t.throwsAsync(got('unix:'), { + await t.throwsAsync(got('unix:', {retry: 0}), { instanceOf: got.RequestError, code: 'ENOTFOUND' });