diff --git a/benchmark/index.ts b/benchmark/index.ts new file mode 100644 index 000000000..0ccca0b3d --- /dev/null +++ b/benchmark/index.ts @@ -0,0 +1,200 @@ +'use strict'; +import {URL} from 'url'; +import https = require('https'); +import axios from 'axios'; +import Benchmark = require('benchmark'); +import fetch from 'node-fetch'; +import request = require('request'); +import got from '../source'; +import PromisableRequest from '../source/as-promise/core'; +import Request, {kIsNormalizedAlready} from '../source/core'; + +const {normalizeArguments} = PromisableRequest; + +// Configuration +const httpsAgent = new https.Agent({ + keepAlive: true, + rejectUnauthorized: false +}); + +const url = new URL('https://127.0.0.1:8080'); +const urlString = url.toString(); + +const gotOptions = { + agent: { + https: httpsAgent + }, + rejectUnauthorized: false, + retry: 0 +}; + +const normalizedGotOptions = normalizeArguments(url, gotOptions); +normalizedGotOptions[kIsNormalizedAlready] = true; + +const requestOptions = { + strictSSL: false, + agent: httpsAgent +}; + +const fetchOptions = { + agent: httpsAgent +}; + +const axiosOptions = { + url: urlString, + httpsAgent, + rejectUnauthorized: false +}; + +const axiosStreamOptions: typeof axiosOptions & {responseType: 'stream'} = { + ...axiosOptions, + responseType: 'stream' +}; + +const httpsOptions = { + rejectUnauthorized: false, + agent: httpsAgent +}; + +const suite = new Benchmark.Suite(); + +// Benchmarking +suite.add('got - promise', { + defer: true, + fn: async (deferred: {resolve(): void}) => { + await got(url, gotOptions); + deferred.resolve(); + } +}).add('got - stream', { + defer: true, + fn: async (deferred: {resolve(): void}) => { + got.stream(url, gotOptions).resume().once('end', () => { + deferred.resolve(); + }); + } +}).add('got - promise core', { + defer: true, + fn: async (deferred: {resolve(): void}) => { + const stream = new PromisableRequest(url, gotOptions); + stream.resume().once('end', () => { + deferred.resolve(); + }); + } +}).add('got - stream core', { + defer: true, + fn: async (deferred: {resolve(): void}) => { + const stream = new Request(url, gotOptions); + stream.resume().once('end', () => { + deferred.resolve(); + }); + } +}).add('got - stream core - normalized options', { + defer: true, + fn: async (deferred: {resolve(): void}) => { + const stream = new Request(undefined as any, normalizedGotOptions); + stream.resume().once('end', () => { + deferred.resolve(); + }); + } +}).add('request - callback', { + defer: true, + fn: (deferred: {resolve(): void}) => { + request(urlString, requestOptions, (error: Error) => { + if (error) { + throw error; + } + + deferred.resolve(); + }); + } +}).add('request - stream', { + defer: true, + fn: (deferred: {resolve(): void}) => { + const stream = request(urlString, requestOptions); + stream.resume(); + stream.once('end', () => { + deferred.resolve(); + }); + } +}).add('node-fetch - promise', { + defer: true, + fn: async (deferred: {resolve(): void}) => { + const response = await fetch(url, fetchOptions); + await response.text(); + + deferred.resolve(); + } +}).add('node-fetch - stream', { + defer: true, + fn: async (deferred: {resolve(): void}) => { + const {body} = await fetch(url, fetchOptions); + + body.resume(); + body.once('end', () => { + deferred.resolve(); + }); + } +}).add('axios - promise', { + defer: true, + fn: async (deferred: {resolve(): void}) => { + await axios.request(axiosOptions); + deferred.resolve(); + } +}).add('axios - stream', { + defer: true, + fn: async (deferred: {resolve(): void}) => { + const {data} = await axios.request(axiosStreamOptions); + data.resume(); + data.once('end', () => { + deferred.resolve(); + }); + } +}).add('https - stream', { + defer: true, + fn: (deferred: {resolve(): void}) => { + https.request(urlString, httpsOptions, response => { + response.resume(); + response.once('end', () => { + deferred.resolve(); + }); + }).end(); + } +}).on('cycle', (event: Benchmark.Event) => { + console.log(String(event.target)); +}).on('complete', function (this: any) { + console.log(`Fastest is ${this.filter('fastest').map('name') as string}`); + + internalBenchmark(); +}).run(); + +const internalBenchmark = (): void => { + console.log(); + + const internalSuite = new Benchmark.Suite(); + internalSuite.add('got - normalize options', { + fn: () => { + normalizeArguments(url, gotOptions); + } + }).on('cycle', (event: Benchmark.Event) => { + console.log(String(event.target)); + }); + + internalSuite.run(); +}; + +// Results (i7-7700k, CPU governor: performance): +// got - promise x 3,092 ops/sec ±5.25% (73 runs sampled) +// got - stream x 4,313 ops/sec ±5.61% (72 runs sampled) +// got - promise core x 6,756 ops/sec ±5.32% (80 runs sampled) +// got - stream core x 6,863 ops/sec ±4.68% (76 runs sampled) +// got - stream core - normalized options x 7,960 ops/sec ±3.83% (81 runs sampled) +// request - callback x 6,912 ops/sec ±6.50% (76 runs sampled) +// request - stream x 7,821 ops/sec ±4.28% (80 runs sampled) +// node-fetch - promise x 7,036 ops/sec ±8.17% (78 runs sampled) +// node-fetch - stream x 7,877 ops/sec ±4.17% (80 runs sampled) +// axios - promise x 6,613 ops/sec ±3.22% (76 runs sampled) +// axios - stream x 8,642 ops/sec ±2.84% (79 runs sampled) +// https - stream x 9,955 ops/sec ±6.36% (76 runs sampled) +// Fastest is https - stream + +// got - normalize options x 166,389 ops/sec ±0.63% (91 runs sampled) diff --git a/benchmark/server.ts b/benchmark/server.ts new file mode 100644 index 000000000..68fd3b10b --- /dev/null +++ b/benchmark/server.ts @@ -0,0 +1,16 @@ +import {AddressInfo} from 'net'; +import https = require('https'); +// @ts-ignore No types +import createCert = require('create-cert'); + +(async () => { + const keys = await createCert({days: 365, commonName: 'localhost'}); + + const server = https.createServer(keys, (_request, response) => { + response.end('ok'); + }).listen(8080, () => { + const {port} = server.address() as AddressInfo; + + console.log(`Listening at https://localhost:${port}`); + }); +})(); diff --git a/documentation/migration-guides.md b/documentation/migration-guides.md index 56b441183..595262563 100644 --- a/documentation/migration-guides.md +++ b/documentation/migration-guides.md @@ -141,9 +141,9 @@ Hooks are powerful, aren't they? [Read more](../readme.md#hooks) to see what els Let's take a quick look at another example from Request's readme: ```js -http.createServer((request, response) => { - if (request.url === '/doodle.png') { - request.pipe(request('https://example.com/doodle.png')).pipe(response); +http.createServer((serverRequest, serverResponse) => { + if (serverRequest.url === '/doodle.png') { + serverRequest.pipe(request('https://example.com/doodle.png')).pipe(serverResponse); } }); ``` @@ -157,15 +157,15 @@ const got = require('got'); const pipeline = promisify(stream.pipeline); -http.createServer(async (request, response) => { - if (request.url === '/doodle.png') { +http.createServer(async (serverRequest, serverResponse) => { + if (serverRequest.url === '/doodle.png') { // When someone makes a request to our server, we receive a body and some headers. // These are passed to Got. Got proxies downloaded data to our server response, // so you don't have to do `response.writeHead(statusCode, headers)` and `response.end(body)`. // It's done automatically. await pipeline( got.stream('https://example.com/doodle.png'), - response + serverResponse ); } }); diff --git a/package.json b/package.json index 99b44128f..8704a0798 100644 --- a/package.json +++ b/package.json @@ -7,10 +7,10 @@ "funding": "https://github.com/sindresorhus/got?sponsor=1", "main": "dist/source", "engines": { - "node": ">=10" + "node": ">=10.19.0" }, "scripts": { - "test": "xo && tsc --noEmit && nyc ava", + "test": "xo && tsc --noEmit && nyc --reporter=html --reporter=text ava", "release": "np", "build": "del-cli dist && tsc", "prepare": "npm run build" @@ -33,45 +33,45 @@ "fetch", "net", "network", - "electron", + "gzip", "brotli", "requests", "human-friendly", "axios", - "superagent" + "superagent", + "node-fetch", + "ky" ], "dependencies": { - "@sindresorhus/is": "^2.0.0", + "@sindresorhus/is": "^2.1.0", "@szmarczak/http-timer": "^4.0.0", "@types/cacheable-request": "^6.0.1", - "@types/keyv": "3.1.1", - "@types/responselike": "1.0.0", - "cacheable-lookup": "^2.0.0", + "@types/responselike": "^1.0.0", + "cacheable-lookup": "^4.1.1", "cacheable-request": "^7.0.1", "decompress-response": "^5.0.0", - "duplexer3": "^0.1.4", "get-stream": "^5.0.0", + "http2-wrapper": "^1.0.0-beta.4.3", "lowercase-keys": "^2.0.0", - "mimic-response": "^2.1.0", "p-cancelable": "^2.0.0", - "p-event": "^4.0.0", - "responselike": "^2.0.0", - "to-readable-stream": "^2.0.0", - "type-fest": "^0.10.0" + "responselike": "^2.0.0" }, "devDependencies": { "@ava/typescript": "^1.1.1", "@sindresorhus/tsconfig": "^0.7.0", - "@types/duplexer3": "^0.1.0", + "@types/benchmark": "^1.0.31", "@types/express": "^4.17.2", "@types/lolex": "^5.1.0", "@types/node": "13.1.2", - "@types/proxyquire": "^1.3.28", + "@types/node-fetch": "^2.5.5", + "@types/request": "^2.48.4", "@types/sinon": "^7.0.13", "@types/tough-cookie": "^2.3.5", "@typescript-eslint/eslint-plugin": "^2.19.2", "@typescript-eslint/parser": "^2.19.2", "ava": "^3.3.0", + "axios": "^0.19.2", + "benchmark": "^2.1.4", "coveralls": "^3.0.4", "create-test-server": "^3.0.1", "del-cli": "^3.0.0", @@ -79,25 +79,22 @@ "eslint-config-xo-typescript": "^0.26.0", "express": "^4.17.1", "form-data": "^3.0.0", - "get-port": "^5.0.0", - "keyv": "^4.0.0", "lolex": "^6.0.0", "nock": "^12.0.0", + "node-fetch": "^2.6.0", "np": "^6.0.0", "nyc": "^15.0.0", - "proxyquire": "^2.0.1", + "p-event": "^4.0.0", "sinon": "^8.1.1", "slow-stream": "0.0.4", "tempy": "^0.4.0", + "to-readable-stream": "^2.1.0", "tough-cookie": "^3.0.0", "typescript": "3.7.5", - "xo": "^0.26.0" + "xo": "^0.26.1" }, "types": "dist/source", "sideEffects": false, - "browser": { - "electron": false - }, "ava": { "files": [ "test/*" diff --git a/readme.md b/readme.md index 1cbe1951b..f112f9511 100644 --- a/readme.md +++ b/readme.md @@ -28,7 +28,8 @@ For browser usage, we recommend [Ky](https://github.com/sindresorhus/ky) by the - [Promise API](#api) - [Stream API](#streams) -- [Pagination API (experimental)](#pagination) +- [Pagination API](#pagination) +- [HTTP2 support](#http2) - [Request cancelation](#aborting-the-request) - [RFC compliant caching](#cache-adapters) - [Follows redirects](#followredirect) @@ -44,8 +45,8 @@ For browser usage, we recommend [Ky](https://github.com/sindresorhus/ky) by the - [Types](#types) - [Composable](documentation/advanced-creation.md#merging-instances) - [Plugins](documentation/lets-make-a-plugin.md) -- [Used by 3000+ packages and 1.6M+ repos](https://github.com/sindresorhus/got/network/dependents) -- Actively maintained +- [Used by 4K+ packages and 1.8M+ repos](https://github.com/sindresorhus/got/network/dependents) +- [Actively maintained](https://github.com/sindresorhus/got/graphs/contributors) ## Install @@ -94,17 +95,17 @@ const pipeline = promisify(stream.pipeline); })(); ``` -**Tip:** Using `from.pipe(to)` doesn't forward errors. If you use it, switch to [`Stream.pipeline(from, ..., to, callback)`](https://nodejs.org/api/stream.html#stream_stream_pipeline_streams_callback) instead (available from Node v10). +**Tip:** `from.pipe(to)` doesn't forward errors. Instead, use [`stream.pipeline(from, ..., to, callback)`](https://nodejs.org/api/stream.html#stream_stream_pipeline_streams_callback). ### API -It's a `GET` request by default, but can be changed by using different methods or via `options.method`. +It's a `GET` request by default, but can be changed by using different methods or via [`options.method`](#method). **By default, Got will retry on failure. To disable this option, set [`options.retry`](#retry) to `0`.** #### got(url?, options?) -Returns a Promise for a [`response` object](#response) or a [stream](#streams-1) if `options.isStream` is set to true. +Returns a Promise giving a [Response object](#response) or a [Got Stream](#streams-1) if `options.isStream` is set to true. ##### url @@ -116,8 +117,6 @@ Properties from `options` will override properties in the parsed `url`. If no protocol is specified, it will throw a `TypeError`. -**Note:** this can also be an option. - ##### options Type: `object` @@ -126,17 +125,25 @@ Any of the [`https.request`](https://nodejs.org/api/https.html#https_https_reque **Note:** Legacy URL support is disabled. `options.path` is supported only for backwards compatibility. Use `options.pathname` and `options.searchParams` instead. `options.auth` has been replaced with `options.username` & `options.password`. +###### method + +Type: `string`\ +Default: `GET` + +The HTTP method used to make the request. + ###### prefixUrl Type: `string | URL` -When specified, `prefixUrl` will be prepended to `url`. The prefix can be any valid URL, either relative or absolute. A trailing slash `/` is optional - one will be added automatically. +When specified, `prefixUrl` will be prepended to `url`. The prefix can be any valid URL, either relative or absolute.\ +A trailing slash `/` is optional - one will be added automatically. **Note:** `prefixUrl` will be ignored if the `url` argument is a URL instance. **Note:** Leading slashes in `input` are disallowed when using this option to enforce consistency and avoid confusion. For example, when the prefix URL is `https://example.com/foo` and the input is `/bar`, there's ambiguity whether the resulting URL would become `https://example.com/foo/bar` or `https://example.com/bar`. The latter is used by browsers. -**Tip:** Useful when used with [`got.extend()`](#custom-endpoints) to create niche-specific Got-instances. +**Tip:** Useful when used with [`got.extend()`](#custom-endpoints) to create niche-specific Got instances. **Tip:** You can change `prefixUrl` using hooks as long as the URL still includes the `prefixUrl`. If the URL doesn't include it anymore, it will throw. @@ -198,7 +205,7 @@ The `content-length` header will be automatically set if `body` is a `string` / Type: `object | Array | number | string | boolean | null` *(JSON-serializable values)* -**Note #1:** If you provide this option, `got.stream()` will be read-only. +**Note #1:** If you provide this option, `got.stream()` will be read-only.\ **Note #2:** This option is not enumerable and will not be merged with the instance defaults. JSON body. If the `Content-Type` header is not set, it will be set to `application/json`. @@ -320,16 +327,18 @@ Default: `'utf8'` To get a [`Buffer`](https://nodejs.org/api/buffer.html), you need to set [`responseType`](#responseType) to `buffer` instead. +**Note:** This doesn't affect streams! Instead, you need to do `got.stream(...).setEncoding(encoding)`. + ###### form -Type: `object | true` +Type: `object` -**Note #1:** If you provide this option, `got.stream()` will be read-only. +**Note #1:** If you provide this option, `got.stream()` will be read-only.\ **Note #2:** This option is not enumerable and will not be merged with the instance defaults. -The form body is converted to query string using [`(new URLSearchParams(object)).toString()`](https://nodejs.org/api/url.html#url_constructor_new_urlsearchparams_obj). +The form body is converted to a query string using [`(new URLSearchParams(object)).toString()`](https://nodejs.org/api/url.html#url_constructor_new_urlsearchparams_obj). -If set to `true` and the `Content-Type` header is not set, it will be set to `application/x-www-form-urlencoded`. +If the `Content-Type` header is not present, it will be set to `application/x-www-form-urlencoded`. ###### searchParams @@ -350,20 +359,6 @@ console.log(searchParams.toString()); //=> 'key=a&key=b' ``` -And if you need a different array format, you could use the [`query-string`](https://github.com/sindresorhus/query-string) package: - -```js -const got = require('got'); -const queryString = require('query-string'); - -const searchParams = queryString.stringify({key: ['a', 'b']}, {arrayFormat: 'bracket'}); - -got('https://example.com', {searchParams}); - -console.log(searchParams); -//=> 'key[]=a&key[]=b' -``` - ###### timeout Type: `number | object` @@ -465,25 +460,38 @@ Default: `false` ###### dnsCache Type: `object`\ -Default: `false` +Default: `new CacheableLookup()` -[Cache adapter instance](#cache-adapters) for storing cached DNS data. +An instance of [`CacheableLookup`](https://github.com/szmarczak/cacheable-lookup) used for making DNS lookups. ###### request Type: `Function`\ Default: `http.request | https.request` *(Depending on the protocol)* -Custom request function. The main purpose of this is to [support HTTP2 using a wrapper](#experimental-http2-support). +Custom request function. The main purpose of this is to [support HTTP2 using a wrapper](https://github.com/szmarczak/http2-wrapper). -###### useElectronNet +###### http2 Type: `boolean`\ Default: `false` -[**Deprecated**](https://github.com/sindresorhus/got#electron-support-has-been-deprecated) +If set to `true`, Got will additionally accept HTTP2 requests.\ +It will choose either HTTP/1.1 or HTTP/2 depending on the ALPN protocol. -When used in Electron, Got will use [`electron.net`](https://electronjs.org/docs/api/net/) instead of the Node.js `http` module. According to the Electron docs, it should be fully compatible, but it's not entirely. See [#443](https://github.com/sindresorhus/got/issues/443) and [#461](https://github.com/sindresorhus/got/issues/461). +**Note:** Overriding `options.request` will disable HTTP2 support. + +**Note:** This option will default to `true` in the next upcoming major release. + +```js +const got = require('got'); + +(async () => { + const {headers} = await got('https://nghttp2.org/httpbin/anything', {http2: true}); + console.log(headers.via); + //=> '2 nghttpx' +})(); +``` ###### throwHttpErrors @@ -496,9 +504,11 @@ If this is disabled, requests that encounter an error status code will be resolv ###### agent -Same as the [`agent` option](https://nodejs.org/api/http.html#http_http_request_url_options_callback) for `http.request`, but with an extra feature: +Type: `object` + +An object representing `http`, `https` and `http2` keys for [`http.Agent`](https://nodejs.org/api/http.html#http_class_http_agent), [`https.Agent`](https://nodejs.org/api/https.html#https_class_https_agent) and [`http2wrapper.Agent`](https://github.com/szmarczak/http2-wrapper#new-http2agentoptions) instance. This is necessary because a request to one protocol might redirect to another. In such a scenario, Got will switch over to the right protocol agent for you. -If you require different agents for different protocols, you can pass a map of agents to the `agent` option. This is necessary because a request to one protocol might redirect to another. In such a scenario, Got will switch over to the right protocol agent for you. +If a key is not present, it will default to a global agent. ```js const got = require('got'); @@ -528,14 +538,16 @@ Called with plain [request options](#options), right before their normalization. See the [Request migration guide](documentation/migration-guides.md#breaking-changes) for an example. -**Note:** This hook must be synchronous! +**Note #1:** This hook must be synchronous!\ +**Note #2:** Errors in this hook will be converted into an instances of [`RequestError`](#got.requesterror).\ +**Note #3:** The options object may not have a `url` property. To modify it, use a `beforeRequest` hook instead. ###### hooks.beforeRequest Type: `Function[]`\ Default: `[]` -Called with [normalized](source/normalize-arguments.ts) [request options](#options). Got will make no further changes to the request before it is sent (except the body serialization). This is especially useful in conjunction with [`got.extend()`](#instances) when you want to create an API client that, for example, uses HMAC-signing. +Called with [normalized](source/core/index.ts) [request options](#options). Got will make no further changes to the request before it is sent. This is especially useful in conjunction with [`got.extend()`](#instances) when you want to create an API client that, for example, uses HMAC-signing. See the [AWS section](#aws) for an example. @@ -544,7 +556,7 @@ See the [AWS section](#aws) for an example. Type: `Function[]`\ Default: `[]` -Called with [normalized](source/normalize-arguments.ts) [request options](#options) and the redirect [response](#response). Got will make no further changes to the request. This is especially useful when you want to avoid dead sites. Example: +Called with [normalized](source/core/index.ts) [request options](#options) and the redirect [response](#response). Got will make no further changes to the request. This is especially useful when you want to avoid dead sites. Example: ```js const got = require('got'); @@ -664,20 +676,20 @@ got('https://api.github.com/some-endpoint', { }); ``` -##### \_pagination +##### pagination Type: `object` -**Note:** This feature is marked as experimental as we're [looking for feedback](https://github.com/sindresorhus/got/issues/1052) on the API and how it works. The feature itself is stable, but the API may change based on feedback. So if you decide to try it out, we suggest locking down the `got` dependency semver range or use a lockfile. +**Note:** We're [looking for feedback](https://github.com/sindresorhus/got/issues/1052), any ideas on how to improve the API are welcome. -###### \_pagination.transform +###### pagination.transform Type: `Function`\ Default: `response => JSON.parse(response.body)` A function that transform [`Response`](#response) into an array of items. This is where you should do the parsing. -###### \_pagination.paginate +###### pagination.paginate Type: `Function`\ Default: [`Link` header logic](source/index.ts) @@ -702,7 +714,7 @@ const got = require('got'); limit, offset: 0 }, - _pagination: { + pagination: { paginate: (response, allItems, currentItems) => { const previousSearchParams = response.request.options.searchParams; const {offset: previousOffset} = previousSearchParams; @@ -725,14 +737,14 @@ const got = require('got'); })(); ``` -###### \_pagination.filter +###### pagination.filter Type: `Function`\ Default: `(item, allItems, currentItems) => true` Checks whether the item should be emitted or not. -###### \_pagination.shouldContinue +###### pagination.shouldContinue Type: `Function`\ Default: `(item, allItems, currentItems) => true` @@ -741,13 +753,19 @@ Checks whether the pagination should continue. For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems, currentItems) => !item.flag`. If you want to stop **after** emitting the entry, you should use `(item, allItems, currentItems) => allItems.some(entry => entry.flag)` instead. -###### \_pagination.countLimit +###### pagination.countLimit Type: `number`\ Default: `Infinity` The maximum amount of items that should be emitted. +##### localAddress + +Type: `string` + +The IP address used to send the request from. + #### Response The response object will typically be a [Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage), however, if returned from the cache it will be a [response-like object](https://github.com/lukechilds/responselike) which behaves in the same way. @@ -866,7 +884,9 @@ The `response` event to get the response object of the final request. The `redirect` event to get the response object of a redirect. The second argument is options for the next request to the redirect location. ##### .on('uploadProgress', progress) +##### .uploadProgress ##### .on('downloadProgress', progress) +##### .downloadProgress Progress events for uploading (sending a request) and downloading (receiving a response). The `progress` argument is an object like: @@ -894,9 +914,29 @@ If the `content-length` header is missing, `total` will be `undefined`. })(); ``` -##### .on('error', error, body, response) +##### .ip -The `error` event emitted in case of a protocol error (like `ENOTFOUND` etc.) or status error (4xx or 5xx). The second argument is the body of the server response in case of status error. The third argument is a response object. +Type: `string` + +The remote IP address. + +##### .aborted + +Type: `boolean` + +Indicates whether the request has been aborted or not. + +##### .timings + +The same as `response.timings`. + +##### .isFromCache + +The same as `response.isFromCache`. + +##### .on('error', error) + +The emitted `error` is an instance of [`RequestError`](#got.requesterror). #### Pagination @@ -909,7 +949,7 @@ Returns an async iterator: const countLimit = 10; const pagination = got.paginate('https://api.github.com/repos/sindresorhus/got/commits', { - _pagination: {countLimit} + pagination: {countLimit} }); console.log(`Printing latest ${countLimit} Got commits (newest to oldest):`); @@ -920,7 +960,7 @@ Returns an async iterator: })(); ``` -See [`options._pagination`](#_pagination) for more pagination options. +See [`options.pagination`](#pagination) for more pagination options. #### got.get(url, options?) #### got.post(url, options?) @@ -929,7 +969,7 @@ See [`options._pagination`](#_pagination) for more pagination options. #### got.head(url, options?) #### got.delete(url, options?) -Sets `options.method` to the method name and makes a request. +Sets [`options.method`](#method) to the method name and makes a request. ### Instances @@ -1167,7 +1207,8 @@ const addAccessToken = (accessToken: string): BeforeRequestHook => options => { ## Errors -Each error contains an `options` property which are the options Got used to create a request - just to make debugging easier. +Each error contains an `options` property which are the options Got used to create a request - just to make debugging easier.\ +Additionaly, the errors may have `request` (Got Stream) and `response` (Got Response) properties depending on which phase of the request failed. #### got.CacheError @@ -1258,6 +1299,15 @@ const got = require('got'); })(); ``` +To abort the Got Stream request, just call `stream.destroy()`. + +```js +const got = require('got'); + +const stream = got.stream(url); +stream.destroy(); +``` + ## Cache @@ -1572,42 +1622,26 @@ const custom = got.extend({ })(); ``` -### Experimental HTTP2 support - -Got provides an experimental support for HTTP2 using the [`http2-wrapper`](https://github.com/szmarczak/http2-wrapper) package: - -```js -const got = require('got'); -const {request} = require('http2-wrapper'); - -const h2got = got.extend({request}); - -(async () => { - const {body} = await h2got('https://nghttp2.org/httpbin/headers'); - console.log(body); -})(); -``` - ## FAQ ### Why yet another HTTP client? Got was created because the popular [`request`](https://github.com/request/request) package is bloated: [![Install size](https://packagephobia.now.sh/badge?p=request)](https://packagephobia.now.sh/result?p=request)\ -Furthermore, Got is fully written in TypeScript. +Furthermore, Got is fully written in TypeScript and actively maintained. -### Electron support has been deprecated +### Electron support has been removed -Some of the Got features may not work properly. See [#899](https://github.com/sindresorhus/got/issues/899) for more info. +The Electron `net` module is not consistent with the Node.js `http` module. See [#899](https://github.com/sindresorhus/got/issues/899) for more info. ## Comparison | | `got` | [`request`][r0] | [`node-fetch`][n0] | [`ky`][k0] | [`axios`][a0] | [`superagent`][s0] | |-----------------------|:------------------:|:------------------:|:--------------------:|:------------------------:|:------------------:|:----------------------:| -| HTTP/2 support | :grey_question: | :x: | :x: | :x: | :x: | :heavy_check_mark:\*\* | +| HTTP/2 support | :sparkle: | :x: | :x: | :x: | :x: | :heavy_check_mark:\*\* | | Browser support | :x: | :x: | :heavy_check_mark:\* | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | | Promise API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | | Stream API | :heavy_check_mark: | :heavy_check_mark: | Node.js only | :x: | :x: | :heavy_check_mark: | -| Pagination API | :sparkle: | :x: | :x: | :x: | :x: | :x: | +| Pagination API | :heavy_check_mark: | :x: | :x: | :x: | :x: | :x: | | Request cancelation | :heavy_check_mark: | :x: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | | RFC compliant caching | :heavy_check_mark: | :x: | :x: | :x: | :x: | :x: | | Cookies (out-of-box) | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :x: | :x: | @@ -1790,14 +1824,8 @@ Some of the Got features may not work properly. See [#899](https://github.com/si - [Brandon Smith](https://github.com/brandon93s) - [Luke Childs](https://github.com/lukechilds) ---- +## For enterprise -
- - Get professional support for this package with a Tidelift subscription - -
- - Tidelift helps make open source sustainable for maintainers while giving companies
assurances about security, maintenance, and licensing for their dependencies. - -
+Available as part of the Tidelift Subscription. + +The maintainers of `got` and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source dependencies you use to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. [Learn more.](https://tidelift.com/subscription/pkg/npm-got?utm_source=npm-got&utm_medium=referral&utm_campaign=enterprise&utm_term=repo) diff --git a/source/as-promise.ts b/source/as-promise.ts deleted file mode 100644 index 9f4f2feac..000000000 --- a/source/as-promise.ts +++ /dev/null @@ -1,182 +0,0 @@ -import EventEmitter = require('events'); -import getStream = require('get-stream'); -import PCancelable = require('p-cancelable'); -import is from '@sindresorhus/is'; -import {ParseError, ReadError, HTTPError} from './errors'; -import {normalizeArguments, mergeOptions} from './normalize-arguments'; -import requestAsEventEmitter, {proxyEvents} from './request-as-event-emitter'; -import {CancelableRequest, GeneralError, NormalizedOptions, Response} from './types'; - -const parseBody = (body: Buffer, responseType: NormalizedOptions['responseType'], encoding: NormalizedOptions['encoding']): unknown => { - if (responseType === 'json') { - return body.length === 0 ? '' : JSON.parse(body.toString()); - } - - if (responseType === 'buffer') { - return Buffer.from(body); - } - - if (responseType === 'text') { - return body.toString(encoding); - } - - throw new TypeError(`Unknown body type '${responseType as string}'`); -}; - -export function createRejection(error: Error): CancelableRequest { - const promise = Promise.reject(error) as CancelableRequest; - const returnPromise = (): CancelableRequest => promise; - - promise.json = returnPromise; - promise.text = returnPromise; - promise.buffer = returnPromise; - promise.on = returnPromise; - - return promise; -} - -export default function asPromise(options: NormalizedOptions): CancelableRequest { - const proxy = new EventEmitter(); - let body: Buffer; - - const promise = new PCancelable((resolve, reject, onCancel) => { - const emitter = requestAsEventEmitter(options); - onCancel(emitter.abort); - - const emitError = async (error: GeneralError): Promise => { - try { - for (const hook of options.hooks.beforeError) { - // eslint-disable-next-line no-await-in-loop - error = await hook(error); - } - - reject(error); - } catch (error_) { - reject(error_); - } - }; - - emitter.on('response', async (response: Response) => { - proxy.emit('response', response); - - // Download body - try { - body = await getStream.buffer(response, {encoding: 'binary'}); - } catch (error) { - emitError(new ReadError(error, options)); - return; - } - - if (response.req?.aborted) { - // Canceled while downloading - will throw a `CancelError` or `TimeoutError` error - return; - } - - const isOk = (): boolean => { - const {statusCode} = response; - const limitStatusCode = options.followRedirect ? 299 : 399; - - return (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304; - }; - - // Parse body - try { - response.body = parseBody(body, options.responseType, options.encoding); - } catch (error) { - // Fall back to `utf8` - response.body = body.toString(); - - if (isOk()) { - const parseError = new ParseError(error, response, options); - emitError(parseError); - return; - } - } - - try { - for (const [index, hook] of options.hooks.afterResponse.entries()) { - // @ts-ignore TS doesn't notice that CancelableRequest is a Promise - // eslint-disable-next-line no-await-in-loop - response = await hook(response, async (updatedOptions): CancelableRequest => { - const typedOptions = normalizeArguments(mergeOptions(options, { - ...updatedOptions, - retry: { - calculateDelay: () => 0 - }, - throwHttpErrors: false, - resolveBodyOnly: false - })); - - // Remove any further hooks for that request, because we'll call them anyway. - // The loop continues. We don't want duplicates (asPromise recursion). - typedOptions.hooks.afterResponse = options.hooks.afterResponse.slice(0, index); - - for (const hook of options.hooks.beforeRetry) { - // eslint-disable-next-line no-await-in-loop - await hook(typedOptions); - } - - const promise = asPromise(typedOptions); - - onCancel(() => { - promise.catch(() => {}); - promise.cancel(); - }); - - return promise as unknown as CancelableRequest; - }); - } - } catch (error) { - emitError(error); - return; - } - - // Check for HTTP error codes - if (!isOk()) { - const error = new HTTPError(response, options); - - if (emitter.retry(error)) { - return; - } - - if (options.throwHttpErrors) { - emitError(error); - return; - } - } - - resolve(options.resolveBodyOnly ? response.body : response); - }); - - emitter.once('error', reject); - - proxyEvents(proxy, emitter); - }) as CancelableRequest; - - promise.on = (name: string, fn: (...args: any[]) => void) => { - proxy.on(name, fn); - return promise; - }; - - const shortcut = (responseType: NormalizedOptions['responseType']): CancelableRequest => { - // eslint-disable-next-line promise/prefer-await-to-then - const newPromise = promise.then(() => parseBody(body, responseType, options.encoding)); - - Object.defineProperties(newPromise, Object.getOwnPropertyDescriptors(promise)); - - return newPromise as CancelableRequest; - }; - - promise.json = () => { - if (is.undefined(body) && is.undefined(options.headers.accept)) { - options.headers.accept = 'application/json'; - } - - return shortcut('json'); - }; - - promise.buffer = () => shortcut('buffer'); - promise.text = () => shortcut('text'); - - return promise; -} diff --git a/source/calculate-retry-delay.ts b/source/as-promise/calculate-retry-delay.ts similarity index 72% rename from source/calculate-retry-delay.ts rename to source/as-promise/calculate-retry-delay.ts index 0dcb65303..34c9e261f 100644 --- a/source/calculate-retry-delay.ts +++ b/source/as-promise/calculate-retry-delay.ts @@ -1,6 +1,10 @@ -import is from '@sindresorhus/is'; -import {HTTPError, ParseError, MaxRedirectsError} from './errors'; -import {RetryFunction, RetryObject} from './types'; +import { + ParseError, + HTTPError, + MaxRedirectsError, + RetryObject, + RetryFunction +} from './types'; const retryAfterStatusCodes: ReadonlySet = new Set([413, 429, 503]); @@ -14,7 +18,7 @@ const calculateRetryDelay: RetryFunction = ({attemptCount, retryOptions, error}) } const hasMethod = retryOptions.methods.includes(error.options.method); - const hasErrorCode = Reflect.has(error, 'code') && retryOptions.errorCodes.includes(error.code); + const hasErrorCode = retryOptions.errorCodes.includes(error.code!); const hasStatusCode = isErrorWithResponse(error) && retryOptions.statusCodes.includes(error.response.statusCode); if (!hasMethod || (!hasErrorCode && !hasStatusCode)) { return 0; @@ -22,15 +26,15 @@ const calculateRetryDelay: RetryFunction = ({attemptCount, retryOptions, error}) if (isErrorWithResponse(error)) { const {response} = error; - if (response && Reflect.has(response.headers, 'retry-after') && retryAfterStatusCodes.has(response.statusCode)) { + if (response && 'retry-after' in response.headers && retryAfterStatusCodes.has(response.statusCode)) { let after = Number(response.headers['retry-after']); - if (is.nan(after)) { + if (isNaN(after)) { after = Date.parse(response.headers['retry-after']!) - Date.now(); } else { after *= 1000; } - if (after > retryOptions.maxRetryAfter) { + if (retryOptions.maxRetryAfter === undefined || after > retryOptions.maxRetryAfter) { return 0; } diff --git a/source/as-promise/core.ts b/source/as-promise/core.ts new file mode 100644 index 000000000..21fe134ad --- /dev/null +++ b/source/as-promise/core.ts @@ -0,0 +1,148 @@ +import {URL} from 'url'; +import is, {assert} from '@sindresorhus/is'; +import { + Options, + NormalizedOptions, + Defaults, + ResponseType +} from './types'; +import Request, {knownHookEvents, RequestError, HTTPError, Method} from '../core'; + +if (!knownHookEvents.includes('beforeRetry' as any)) { + knownHookEvents.push('beforeRetry' as any, 'afterResponse' as any); +} + +export const knownBodyTypes = ['json', 'buffer', 'text']; + +// @ts-ignore The error is: Not all code paths return a value. +export const parseBody = (body: Buffer, responseType: ResponseType, encoding?: string): unknown => { + if (responseType === 'text') { + return body.toString(encoding); + } + + if (responseType === 'json') { + return body.length === 0 ? '' : JSON.parse(body.toString()); + } + + if (responseType === 'buffer') { + return Buffer.from(body); + } + + if (!knownBodyTypes.includes(responseType)) { + throw new TypeError(`Unknown body type '${responseType as string}'`); + } +}; + +export default class PromisableRequest extends Request { + ['constructor']: typeof PromisableRequest; + declare options: NormalizedOptions; + declare _throwHttpErrors: boolean; + + static normalizeArguments(url?: string | URL, nonNormalizedOptions?: Options, defaults?: Defaults): NormalizedOptions { + const options = super.normalizeArguments(url, nonNormalizedOptions, defaults) as NormalizedOptions; + + if (is.null_(options.encoding)) { + throw new TypeError('To get a Buffer, set `options.responseType` to `buffer` instead'); + } + + assert.any([is.string, is.undefined], options.encoding); + assert.any([is.boolean, is.undefined], options.resolveBodyOnly); + assert.any([is.boolean, is.undefined], options.methodRewriting); + assert.any([is.boolean, is.undefined], options.isStream); + + // `options.retry` + const {retry} = options; + + if (defaults) { + options.retry = {...defaults.retry}; + } else { + options.retry = { + calculateDelay: retryObject => retryObject.computedValue, + limit: 0, + methods: [], + statusCodes: [], + errorCodes: [], + maxRetryAfter: undefined + }; + } + + if (is.object(retry)) { + options.retry = { + ...options.retry, + ...retry + }; + + options.retry.methods = [...new Set(options.retry.methods.map(method => method.toUpperCase() as Method))]; + options.retry.statusCodes = [...new Set(options.retry.statusCodes)]; + options.retry.errorCodes = [...new Set(options.retry.errorCodes)]; + } else if (is.number(retry)) { + options.retry.limit = retry; + } + + if (is.undefined(options.retry.maxRetryAfter)) { + options.retry.maxRetryAfter = Math.min( + ...[options.timeout.request, options.timeout.connect].filter(is.number) + ); + } + + // `options.pagination` + if (is.object(options.pagination)) { + if (defaults) { + (options as Options).pagination = { + ...defaults.pagination, + ...options.pagination + }; + } + + const {pagination} = options; + + if (!is.function_(pagination.transform)) { + throw new Error('`options.pagination.transform` must be implemented'); + } + + if (!is.function_(pagination.shouldContinue)) { + throw new Error('`options.pagination.shouldContinue` must be implemented'); + } + + if (!is.function_(pagination.filter)) { + throw new TypeError('`options.pagination.filter` must be implemented'); + } + + if (!is.function_(pagination.paginate)) { + throw new Error('`options.pagination.paginate` must be implemented'); + } + } + + return options; + } + + static mergeOptions(...sources: Options[]): NormalizedOptions { + let mergedOptions: NormalizedOptions | undefined; + + for (const source of sources) { + mergedOptions = PromisableRequest.normalizeArguments(undefined, source, mergedOptions); + } + + return mergedOptions!; + } + + async _beforeError(error: RequestError): Promise { + const isHTTPError = error instanceof HTTPError; + + try { + for (const hook of this.options.hooks.beforeError) { + // eslint-disable-next-line no-await-in-loop + error = await hook(error); + } + } catch (error_) { + this.destroy(error_); + return; + } + + if (this._throwHttpErrors && !isHTTPError) { + this.destroy(error); + } else { + this.emit('error', error); + } + } +} diff --git a/source/as-promise/create-rejection.ts b/source/as-promise/create-rejection.ts new file mode 100644 index 000000000..358e1fe76 --- /dev/null +++ b/source/as-promise/create-rejection.ts @@ -0,0 +1,31 @@ +import {CancelableRequest, BeforeErrorHook, RequestError} from './types'; + +export default function createRejection(error: Error, ...beforeErrorGroups: Array): CancelableRequest { + const promise = (async () => { + if (error instanceof RequestError) { + try { + for (const hooks of beforeErrorGroups) { + if (hooks) { + for (const hook of hooks) { + // eslint-disable-next-line no-await-in-loop + error = await hook(error as RequestError); + } + } + } + } catch (error_) { + error = error_; + } + } + + throw error; + })() as CancelableRequest; + + const returnPromise = (): CancelableRequest => promise; + + promise.json = returnPromise; + promise.text = returnPromise; + promise.buffer = returnPromise; + promise.on = returnPromise; + + return promise; +} diff --git a/source/as-promise/index.ts b/source/as-promise/index.ts new file mode 100644 index 000000000..80133f345 --- /dev/null +++ b/source/as-promise/index.ts @@ -0,0 +1,244 @@ +import {EventEmitter} from 'events'; +import getStream = require('get-stream'); +import PCancelable = require('p-cancelable'); +import calculateRetryDelay from './calculate-retry-delay'; +import { + NormalizedOptions, + CancelableRequest, + Response, + RequestError, + HTTPError, + ReadError, + ParseError +} from './types'; +import PromisableRequest, {parseBody} from './core'; +import proxyEvents from '../core/utils/proxy-events'; + +const proxiedRequestEvents = [ + 'request', + 'response', + 'redirect', + 'uploadProgress', + 'downloadProgress' +]; + +export default function asPromise(options: NormalizedOptions): CancelableRequest { + let retryCount = 0; + let body: Buffer; + const emitter = new EventEmitter(); + + const promise = new PCancelable((resolve, reject, onCancel) => { + const makeRequest = (): void => { + if (options.responseType === 'json' && options.headers.accept === undefined) { + options.headers.accept = 'application/json'; + } + + // Support retries + const {throwHttpErrors} = options; + if (!throwHttpErrors) { + options.throwHttpErrors = true; + } + + const request = new PromisableRequest(options.url, options); + request._throwHttpErrors = throwHttpErrors; + request._noPipe = true; + onCancel(() => request.destroy()); + + request.once('response', async (response: Response) => { + response.retryCount = retryCount; + + if (response.request.aborted) { + // Canceled while downloading - will throw a `CancelError` or `TimeoutError` error + return; + } + + const isOk = (): boolean => { + const {statusCode} = response; + const limitStatusCode = options.followRedirect ? 299 : 399; + + return (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304; + }; + + // Download body + try { + body = await getStream.buffer(request); + } catch (error) { + request._beforeError(new ReadError(error, options, response)); + return; + } + + // Parse body + try { + response.body = parseBody(body, options.responseType, options.encoding); + } catch (error) { + // Fallback to `utf8` + response.body = body.toString('utf8'); + + if (isOk()) { + const parseError = new ParseError(error, response, options); + request._beforeError(parseError); + return; + } + } + + try { + for (const [index, hook] of options.hooks.afterResponse.entries()) { + // @ts-ignore TS doesn't notice that CancelableRequest is a Promise + // eslint-disable-next-line no-await-in-loop + response = await hook(response, async (updatedOptions): CancelableRequest => { + request.destroy(); + + const typedOptions = PromisableRequest.normalizeArguments(undefined, { + ...updatedOptions, + retry: { + calculateDelay: () => 0 + }, + throwHttpErrors: false, + resolveBodyOnly: false + }, options); + + // Remove any further hooks for that request, because we'll call them anyway. + // The loop continues. We don't want duplicates (asPromise recursion). + typedOptions.hooks.afterResponse = typedOptions.hooks.afterResponse.slice(0, index); + + for (const hook of typedOptions.hooks.beforeRetry) { + // eslint-disable-next-line no-await-in-loop + await hook(typedOptions); + } + + const promise: CancelableRequest = asPromise(typedOptions); + + onCancel(() => { + promise.catch(() => {}); + promise.cancel(); + }); + + return promise; + }); + } + } catch (error) { + request._beforeError(error); + return; + } + + if (throwHttpErrors && !isOk()) { + reject(new HTTPError(response, options)); + return; + } + + resolve(options.resolveBodyOnly ? response.body as T : response as unknown as T); + }); + + request.once('error', (error: RequestError) => { + if (promise.isCanceled) { + return; + } + + if (!request.options) { + reject(error); + return; + } + + let backoff: number; + + retryCount++; + + try { + backoff = options.retry.calculateDelay({ + attemptCount: retryCount, + retryOptions: options.retry, + error, + computedValue: calculateRetryDelay({ + attemptCount: retryCount, + retryOptions: options.retry, + error, + computedValue: 0 + }) + }); + } catch (error_) { + // Don't emit the `response` event + request.destroy(); + + reject(new RequestError(error_.message, error, request.options)); + return; + } + + if (backoff) { + // Don't emit the `response` event + request.destroy(); + + const retry = async (): Promise => { + options.throwHttpErrors = throwHttpErrors; + + try { + for (const hook of options.hooks.beforeRetry) { + // eslint-disable-next-line no-await-in-loop + await hook(options, error, retryCount); + } + } catch (error_) { + // Don't emit the `response` event + request.destroy(); + + reject(new RequestError(error_.message, error, request.options)); + return; + } + + makeRequest(); + }; + + setTimeout(retry, backoff); + return; + } + + // The retry has not been made + retryCount--; + + if (error instanceof HTTPError) { + // It will be handled by the `response` event + return; + } + + // Don't emit the `response` event + request.destroy(); + + reject(error); + }); + + proxyEvents(request, emitter, proxiedRequestEvents); + }; + + makeRequest(); + }) as CancelableRequest; + + promise.on = (event: string, fn: (...args: any[]) => void) => { + emitter.on(event, fn); + return promise; + }; + + const shortcut = (responseType: NormalizedOptions['responseType']): CancelableRequest => { + const newPromise = (async () => { + await promise; + return parseBody(body, responseType); + })(); + + Object.defineProperties(newPromise, Object.getOwnPropertyDescriptors(promise)); + + return newPromise as CancelableRequest; + }; + + promise.json = () => { + if (body === undefined && options.headers.accept === undefined) { + options.headers.accept = 'application/json'; + } + + return shortcut('json'); + }; + + promise.buffer = () => shortcut('buffer'); + promise.text = () => shortcut('text'); + + return promise; +} + +export * from './types'; +export {PromisableRequest}; diff --git a/source/as-promise/types.ts b/source/as-promise/types.ts new file mode 100644 index 000000000..2691422b3 --- /dev/null +++ b/source/as-promise/types.ts @@ -0,0 +1,154 @@ +import PCancelable = require('p-cancelable'); +import {CancelError} from 'p-cancelable'; +import { + // Interfaces to be extended + Options as RequestOptions, + NormalizedOptions as RequestNormalizedOptions, + Defaults as RequestDefaults, + Hooks as RequestHooks, + Response as RequestResponse, + + // Errors to be exported + RequestError, + MaxRedirectsError, + CacheError, + UploadError, + TimeoutError, + HTTPError, + ReadError, + UnsupportedProtocolError, + + // Hooks to be exported + HookEvent as RequestHookEvent, + InitHook, + BeforeRequestHook, + BeforeRedirectHook, + BeforeErrorHook, + + // Other types to be exported + Progress, + Headers, + RequestFunction, + + // Types that will not be exported + Method, + RequestEvents +} from '../core'; +import PromisableRequest from './core'; + +export type ResponseType = 'json' | 'buffer' | 'text'; + +export interface Response extends RequestResponse { + request: PromisableRequest; +} + +export interface RetryObject { + attemptCount: number; + retryOptions: RequiredRetryOptions; + error: TimeoutError | RequestError; + computedValue: number; +} + +export type RetryFunction = (retryObject: RetryObject) => number; + +export interface RequiredRetryOptions { + limit: number; + methods: Method[]; + statusCodes: number[]; + errorCodes: string[]; + calculateDelay: RetryFunction; + maxRetryAfter?: number; +} + +export type BeforeRetryHook = (options: NormalizedOptions, error?: RequestError, retryCount?: number) => void | Promise; +export type AfterResponseHook = (response: Response, retryWithMergedOptions: (options: Options) => CancelableRequest) => Response | CancelableRequest | Promise>; + +export interface Hooks extends RequestHooks { + beforeRetry?: BeforeRetryHook[]; + afterResponse?: AfterResponseHook[]; +} + +export interface PaginationOptions { + pagination?: { + transform?: (response: Response) => Promise | T[]; + filter?: (item: T, allItems: T[], currentItems: T[]) => boolean; + paginate?: (response: Response, allItems: T[], currentItems: T[]) => Options | false; + shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean; + countLimit?: number; + }; +} + +export interface Options extends RequestOptions, PaginationOptions { + hooks?: Hooks; + responseType?: ResponseType; + resolveBodyOnly?: boolean; + retry?: Partial | number; + isStream?: boolean; + encoding?: BufferEncoding; +} + +export interface NormalizedOptions extends RequestNormalizedOptions { + hooks: Required; + responseType: ResponseType; + resolveBodyOnly: boolean; + retry: RequiredRetryOptions; + isStream: boolean; + encoding?: BufferEncoding; + pagination?: Required['pagination']>; +} + +export interface Defaults extends RequestDefaults { + hooks: Required; + responseType: ResponseType; + resolveBodyOnly: boolean; + retry: RequiredRetryOptions; + isStream: boolean; + pagination?: Required['pagination']>; +} + +export class ParseError extends RequestError { + declare readonly response: Response; + + constructor(error: Error, response: Response, options: NormalizedOptions) { + super(`${error.message} in "${options.url.toString()}"`, error, options); + this.name = 'ParseError'; + + Object.defineProperty(this, 'response', { + enumerable: false, + value: response + }); + } +} + +export interface CancelableRequest extends PCancelable, RequestEvents> { + json(): CancelableRequest; + buffer(): CancelableRequest; + text(): CancelableRequest; +} + +export type HookEvent = RequestHookEvent | 'beforeRetry' | 'afterResponse'; + +export { + RequestError, + MaxRedirectsError, + CacheError, + UploadError, + TimeoutError, + HTTPError, + ReadError, + UnsupportedProtocolError, + CancelError +}; + +export { + InitHook, + BeforeRequestHook, + BeforeRedirectHook, + BeforeErrorHook +}; + +export { + Progress, + Headers, + RequestFunction +}; diff --git a/source/as-stream.ts b/source/as-stream.ts deleted file mode 100644 index 4a942abda..000000000 --- a/source/as-stream.ts +++ /dev/null @@ -1,144 +0,0 @@ -import duplexer3 = require('duplexer3'); -import {IncomingMessage, ServerResponse} from 'http'; -import {Duplex as DuplexStream, PassThrough as PassThroughStream} from 'stream'; -import {HTTPError, ReadError} from './errors'; -import requestAsEventEmitter, {proxyEvents} from './request-as-event-emitter'; -import {GeneralError, GotEvents, NormalizedOptions, Response} from './types'; - -export class ProxyStream extends DuplexStream implements GotEvents> { - isFromCache?: boolean; -} - -export default function asStream(options: NormalizedOptions): ProxyStream { - const input = new PassThroughStream(); - const output = new PassThroughStream(); - const proxy = duplexer3(input, output) as ProxyStream; - const piped = new Set(); - let isFinished = false; - - options.retry.calculateDelay = () => 0; - - if (options.body || options.json || options.form) { - proxy.write = () => { - proxy.destroy(); - throw new Error('Got\'s stream is not writable when the `body`, `json` or `form` option is used'); - }; - } else if (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH' || (options.allowGetBody && options.method === 'GET')) { - options.body = input; - } else { - proxy.write = () => { - proxy.destroy(); - throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); - }; - } - - const emitter = requestAsEventEmitter(options); - - const emitError = async (error: GeneralError): Promise => { - try { - for (const hook of options.hooks.beforeError) { - // eslint-disable-next-line no-await-in-loop - error = await hook(error); - } - - proxy.emit('error', error); - } catch (error_) { - proxy.emit('error', error_); - } - }; - - // Cancels the request - proxy._destroy = (error, callback) => { - callback(error); - emitter.abort(); - }; - - emitter.on('response', (response: Response) => { - const {statusCode, isFromCache} = response; - proxy.isFromCache = isFromCache; - - if (options.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > 299)) { - emitError(new HTTPError(response, options)); - return; - } - - { - const read = proxy._read; - proxy._read = (...args) => { - isFinished = true; - - proxy._read = read; - return read.apply(proxy, args); - }; - } - - if (options.encoding) { - proxy.setEncoding(options.encoding); - } - - // We cannot use `stream.pipeline(...)` here, - // because if we did then `output` would throw - // the original error before throwing `ReadError`. - response.pipe(output); - response.once('error', error => { - emitError(new ReadError(error, options)); - }); - - for (const destination of piped) { - if (destination.headersSent) { - continue; - } - - for (const [key, value] of Object.entries(response.headers)) { - // Got gives *decompressed* data. Overriding `content-encoding` header would result in an error. - // It's not possible to decompress already decompressed data, is it? - const isAllowed = options.decompress ? key !== 'content-encoding' : true; - if (isAllowed) { - destination.setHeader(key, value!); - } - } - - destination.statusCode = response.statusCode; - } - - proxy.emit('response', response); - }); - - proxyEvents(proxy, emitter); - emitter.on('error', (error: GeneralError) => proxy.emit('error', error)); - - const pipe = proxy.pipe.bind(proxy); - const unpipe = proxy.unpipe.bind(proxy); - - proxy.pipe = (destination, options) => { - if (isFinished) { - throw new Error('Failed to pipe. The response has been emitted already.'); - } - - pipe(destination, options); - - if (destination instanceof ServerResponse) { - piped.add(destination); - } - - return destination; - }; - - proxy.unpipe = stream => { - piped.delete(stream as ServerResponse); - return unpipe(stream); - }; - - proxy.on('pipe', source => { - if (source instanceof IncomingMessage) { - options.headers = { - ...source.headers, - ...options.headers - }; - } - }); - - proxy.isFromCache = undefined; - - return proxy; -} diff --git a/source/core/index.ts b/source/core/index.ts new file mode 100644 index 000000000..0687c3d5c --- /dev/null +++ b/source/core/index.ts @@ -0,0 +1,1492 @@ +import {promisify} from 'util'; +import {Duplex, Writable, Readable} from 'stream'; +import {ReadStream} from 'fs'; +import {URL, URLSearchParams} from 'url'; +import {Socket} from 'net'; +import {SecureContextOptions} from 'tls'; +import http = require('http'); +import {ClientRequest, RequestOptions, IncomingMessage, ServerResponse, request as httpRequest} from 'http'; +import https = require('https'); +import timer, {ClientRequestWithTimings, Timings, IncomingMessageWithTimings} from '@szmarczak/http-timer'; +import decompressResponse = require('decompress-response'); +import CacheableLookup from 'cacheable-lookup'; +import CacheableRequest = require('cacheable-request'); +// @ts-ignore Missing types +import http2wrapper = require('http2-wrapper'); +import lowercaseKeys = require('lowercase-keys'); +import ResponseLike = require('responselike'); +import getStream = require('get-stream'); +import is, {assert} from '@sindresorhus/is'; +import getBodySize from './utils/get-body-size'; +import isFormData from './utils/is-form-data'; +import proxyEvents from './utils/proxy-events'; +import timedOut, {Delays, TimeoutError as TimedOutTimeoutError} from './utils/timed-out'; +import urlToOptions from './utils/url-to-options'; +import optionsToUrl, {URLOptions} from './utils/options-to-url'; + +type HttpRequestFunction = typeof httpRequest; +type Error = NodeJS.ErrnoException; + +const kRequest = Symbol('request'); +const kResponse = Symbol('response'); +const kResponseSize = Symbol('responseSize'); +const kDownloadedSize = Symbol('downloadedSize'); +const kBodySize = Symbol('bodySize'); +const kUploadedSize = Symbol('uploadedSize'); +const kServerResponsesPiped = Symbol('serverResponsesPiped'); +const kUnproxyEvents = Symbol('unproxyEvents'); +const kIsFromCache = Symbol('isFromCache'); +const kCancelTimeouts = Symbol('cancelTimeouts'); +const kStartedReading = Symbol('startedReading'); +export const kIsNormalizedAlready = Symbol('isNormalizedAlready'); + +const supportsBrotli = is.string((process.versions as any).brotli); + +export interface Agents { + http?: http.Agent; + https?: https.Agent; + http2?: unknown; +} + +export const withoutBody: ReadonlySet = new Set(['GET', 'HEAD']); + +export interface ToughCookieJar { + getCookieString(currentUrl: string, options: {[key: string]: unknown}, cb: (err: Error | null, cookies: string) => void): void; + getCookieString(url: string, callback: (error: Error | null, cookieHeader: string) => void): void; + setCookie(cookieOrString: unknown, currentUrl: string, options: {[key: string]: unknown}, cb: (err: Error | null, cookie: unknown) => void): void; + setCookie(rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void): void; +} + +export interface PromiseCookieJar { + getCookieString(url: string): Promise; + setCookie(rawCookie: string, url: string): Promise; +} + +export type Method = + | 'GET' + | 'POST' + | 'PUT' + | 'PATCH' + | 'HEAD' + | 'DELETE' + | 'OPTIONS' + | 'TRACE' + | 'get' + | 'post' + | 'put' + | 'patch' + | 'head' + | 'delete' + | 'options' + | 'trace'; + +type Promisable = T | Promise; + +export type InitHook = (options: Options) => Promisable; +export type BeforeRequestHook = (options: NormalizedOptions) => Promisable; +export type BeforeRedirectHook = (options: NormalizedOptions, response: Response) => Promisable; +export type BeforeErrorHook = (error: RequestError) => Promisable; + +export interface Hooks { + init?: InitHook[]; + beforeRequest?: BeforeRequestHook[]; + beforeRedirect?: BeforeRedirectHook[]; + beforeError?: BeforeErrorHook[]; +} + +export type HookEvent = 'init' | 'beforeRequest' | 'beforeRedirect' | 'beforeError'; + +export const knownHookEvents: HookEvent[] = ['init', 'beforeRequest', 'beforeRedirect', 'beforeError']; + +type AcceptableResponse = IncomingMessage | ResponseLike; +type AcceptableRequestResult = AcceptableResponse | ClientRequest | Promise | undefined; + +export type RequestFunction = (url: URL, options: RequestOptions, callback?: (response: AcceptableResponse) => void) => AcceptableRequestResult; + +export type Headers = Record; + +export interface Options extends URLOptions, SecureContextOptions { + request?: RequestFunction; + agent?: Agents | false; + decompress?: boolean; + timeout?: Delays | number; + prefixUrl?: string | URL; + body?: string | Buffer | Readable; + form?: {[key: string]: any}; + json?: {[key: string]: any}; + url?: string | URL; + cookieJar?: PromiseCookieJar | ToughCookieJar; + ignoreInvalidCookies?: boolean; + searchParams?: string | {[key: string]: string | number | boolean | null} | URLSearchParams; + dnsCache?: CacheableLookup | boolean; + context?: object; + hooks?: Hooks; + followRedirect?: boolean; + maxRedirects?: number; + cache?: string | CacheableRequest.StorageAdapter | false; + throwHttpErrors?: boolean; + username?: string; + password?: string; + http2?: boolean; + allowGetBody?: boolean; + lookup?: CacheableLookup['lookup']; + rejectUnauthorized?: boolean; + headers?: Headers; + methodRewriting?: boolean; + + // From http.RequestOptions + localAddress?: string; + socketPath?: string; + method?: Method; + createConnection?: (options: http.RequestOptions, oncreate: (error: Error, socket: Socket) => void) => Socket; +} + +export interface NormalizedOptions extends Options { + method: Method; + url: URL; + timeout: Delays; + prefixUrl: string; + ignoreInvalidCookies: boolean; + decompress: boolean; + searchParams?: URLSearchParams; + cookieJar?: PromiseCookieJar; + headers: Headers; + context: object; + hooks: Required; + followRedirect: boolean; + maxRedirects: number; + cache?: string | CacheableRequest.StorageAdapter; + throwHttpErrors: boolean; + dnsCache?: CacheableLookup; + cacheableRequest?: (options: string | URL | http.RequestOptions, callback?: (response: http.ServerResponse | ResponseLike) => void) => CacheableRequest.Emitter; + http2: boolean; + allowGetBody: boolean; + rejectUnauthorized: boolean; + lookup?: CacheableLookup['lookup']; + methodRewriting: boolean; + username: string; + password: string; + [kRequest]: HttpRequestFunction; + [kIsNormalizedAlready]?: boolean; +} + +export interface Defaults { + timeout: Delays; + prefixUrl: string; + method: Method; + ignoreInvalidCookies: boolean; + decompress: boolean; + context: object; + cookieJar?: PromiseCookieJar | ToughCookieJar; + dnsCache?: CacheableLookup; + headers: Headers; + hooks: Required; + followRedirect: boolean; + maxRedirects: number; + cache?: string | CacheableRequest.StorageAdapter; + throwHttpErrors: boolean; + http2: boolean; + allowGetBody: boolean; + rejectUnauthorized: boolean; + methodRewriting: boolean; + + // Optional + agent?: Agents | false; + request?: RequestFunction; + searchParams?: URLSearchParams; + lookup?: CacheableLookup['lookup']; + localAddress?: string; + createConnection?: Options['createConnection']; +} + +export interface Progress { + percent: number; + transferred: number; + total?: number; +} + +export interface PlainResponse extends IncomingMessageWithTimings { + requestUrl: string; + redirectUrls: string[]; + request: Request; + ip?: string; + isFromCache: boolean; + statusCode: number; + url: string; + timings: Timings; +} + +// For Promise support +export interface Response extends PlainResponse { + body: T; + retryCount: number; +} + +export interface RequestEvents { + on(name: 'request', listener: (request: http.ClientRequest) => void): T; + on(name: 'response', listener: (response: R) => void): T; + on(name: 'redirect', listener: (response: R, nextOptions: N) => void): T; + on(name: 'uploadProgress' | 'downloadProgress', listener: (progress: Progress) => void): T; +} + +function validateSearchParams(searchParams: Record): asserts searchParams is Record { + // eslint-disable-next-line guard-for-in + for (const key in searchParams) { + const value = searchParams[key]; + + if (!is.string(value) && !is.number(value) && !is.boolean(value) && !is.null_(value)) { + throw new TypeError(`The \`searchParams\` value '${String(value)}' must be a string, number, boolean or null`); + } + } +} + +function isClientRequest(clientRequest: unknown): clientRequest is ClientRequest { + return is.object(clientRequest) && !('statusCode' in clientRequest); +} + +const cacheFn = async (url: URL, options: RequestOptions): Promise => new Promise((resolve, reject) => { + // TODO: Remove `utils/url-to-options.ts` when `cacheable-request` is fixed + Object.assign(options, urlToOptions(url)); + + // `http-cache-semantics` checks this + delete (options as unknown as NormalizedOptions).url; + + // TODO: `cacheable-request` is incorrectly typed + const cacheRequest = (options as Pick).cacheableRequest!(options, resolve as any); + + // Restore options + (options as unknown as NormalizedOptions).url = url; + + cacheRequest.once('error', (error: Error) => { + if (error instanceof CacheableRequest.RequestError) { + // TODO: `options` should be `normalizedOptions` + reject(new RequestError(error.message, error, options as unknown as NormalizedOptions)); + return; + } + + // TODO: `options` should be `normalizedOptions` + reject(new CacheError(error, options as unknown as NormalizedOptions)); + }); + cacheRequest.once('request', resolve); +}); + +const waitForOpenFile = async (file: ReadStream): Promise => new Promise((resolve, reject) => { + const onError = (error: Error): void => { + reject(error); + }; + + file.once('error', onError); + file.once('open', () => { + file.off('error', onError); + resolve(); + }); +}); + +const redirectCodes: ReadonlySet = new Set([300, 301, 302, 303, 304, 307, 308]); + +type NonEnumerableProperty = 'context' | 'body' | 'json' | 'form'; +const nonEnumerableProperties: NonEnumerableProperty[] = [ + 'context', + 'body', + 'json', + 'form' +]; + +const setNonEnumerableProperties = (sources: Array, to: Options): void => { + // Non enumerable properties shall not be merged + const properties: Partial<{[Key in NonEnumerableProperty]: any}> = {}; + + for (const source of sources) { + if (!source) { + continue; + } + + for (const name of nonEnumerableProperties) { + if (!(name in source)) { + continue; + } + + properties[name] = { + writable: true, + configurable: true, + enumerable: false, + // @ts-ignore TS doesn't see the check above + value: source[name] + }; + } + } + + Object.defineProperties(to, properties); +}; + +export class RequestError extends Error { + code?: string; + stack!: string; + declare readonly options: NormalizedOptions; + readonly response?: Response; + readonly request?: Request; + readonly timings?: Timings; + + constructor(message: string, error: Partial, options: NormalizedOptions, requestOrResponse?: Request | Response) { + super(message); + Error.captureStackTrace(this, this.constructor); + + this.name = 'RequestError'; + this.code = error.code; + + Object.defineProperty(this, 'options', { + // This fails because of TS 3.7.2 useDefineForClassFields + // Ref: https://github.com/microsoft/TypeScript/issues/34972 + enumerable: false, + value: options + }); + + if (requestOrResponse instanceof IncomingMessage) { + Object.defineProperty(this, 'response', { + enumerable: false, + value: requestOrResponse + }); + + Object.defineProperty(this, 'request', { + enumerable: false, + value: requestOrResponse.request + }); + } else if (requestOrResponse instanceof Request) { + Object.defineProperty(this, 'request', { + enumerable: false, + value: requestOrResponse + }); + + Object.defineProperty(this, 'response', { + enumerable: false, + value: requestOrResponse[kResponse] + }); + } + + this.timings = this.request?.timings; + + // Recover the original stacktrace + if (!is.undefined(error.stack)) { + const indexOfMessage = this.stack.indexOf(this.message) + this.message.length; + const thisStackTrace = this.stack.slice(indexOfMessage).split('\n').reverse(); + const errorStackTrace = error.stack.slice(error.stack.indexOf(error.message!) + error.message!.length).split('\n').reverse(); + + // Remove duplicated traces + while (errorStackTrace.length !== 0 && errorStackTrace[0] === thisStackTrace[0]) { + thisStackTrace.shift(); + } + + this.stack = `${this.stack.slice(0, indexOfMessage)}${thisStackTrace.reverse().join('\n')}${errorStackTrace.reverse().join('\n')}`; + } + } +} + +export class MaxRedirectsError extends RequestError { + declare readonly response: Response; + + constructor(response: Response, maxRedirects: number, options: NormalizedOptions) { + super(`Redirected ${maxRedirects} times. Aborting.`, {}, options); + this.name = 'MaxRedirectsError'; + + Object.defineProperty(this, 'response', { + enumerable: false, + value: response + }); + } +} + +export class HTTPError extends RequestError { + declare readonly response: Response; + + constructor(response: Response, options: NormalizedOptions) { + super(`Response code ${response.statusCode} (${response.statusMessage!})`, {}, options); + this.name = 'HTTPError'; + + Object.defineProperty(this, 'response', { + enumerable: false, + value: response + }); + } +} + +export class CacheError extends RequestError { + constructor(error: Error, options: NormalizedOptions) { + super(error.message, error, options); + this.name = 'CacheError'; + } +} + +export class UploadError extends RequestError { + constructor(error: Error, options: NormalizedOptions, request: Request) { + super(error.message, error, options, request); + this.name = 'UploadError'; + } +} + +export class TimeoutError extends RequestError { + readonly timings: Timings; + readonly event: string; + + constructor(error: TimedOutTimeoutError, timings: Timings, options: NormalizedOptions) { + super(error.message, error, options); + this.name = 'TimeoutError'; + this.event = error.event; + this.timings = timings; + } +} + +export class ReadError extends RequestError { + constructor(error: Error, options: NormalizedOptions, response: Response) { + super(error.message, error, options, response); + this.name = 'ReadError'; + } +} + +export class UnsupportedProtocolError extends RequestError { + constructor(options: NormalizedOptions) { + super(`Unsupported protocol "${options.url.protocol}"`, {}, options); + this.name = 'UnsupportedProtocolError'; + } +} + +const proxiedRequestEvents = [ + 'socket', + 'abort', + 'connect', + 'continue', + 'information', + 'upgrade', + 'timeout' +]; + +export default class Request extends Duplex implements RequestEvents { + ['constructor']: typeof Request; + + declare [kUnproxyEvents]: () => void; + declare _cannotHaveBody: boolean; + [kDownloadedSize]: number; + [kUploadedSize]: number; + [kBodySize]?: number; + [kServerResponsesPiped]: Set; + [kIsFromCache]?: boolean; + [kStartedReading]?: boolean; + [kCancelTimeouts]?: () => void; + [kResponseSize]?: number; + [kResponse]?: IncomingMessage; + [kRequest]?: ClientRequest; + _noPipe?: boolean; + _progressCallbacks: Array<() => void>; + + declare options: NormalizedOptions; + declare requestUrl: string; + finalized: boolean; + redirects: string[]; + errored: boolean; + + constructor(url: string | URL, options: Options = {}, defaults?: Defaults) { + super({ + // It needs to be zero because we're just proxying the data to another stream + highWaterMark: 0 + }); + + this[kDownloadedSize] = 0; + this[kUploadedSize] = 0; + this.finalized = false; + this[kServerResponsesPiped] = new Set(); + this.redirects = []; + this.errored = false; + + // TODO: Remove this when targeting Node.js >= 12 + this._progressCallbacks = []; + + const unlockWrite = (): void => this._unlockWrite(); + const lockWrite = (): void => this._lockWrite(); + + this.on('pipe', (source: Writable) => { + source.prependListener('data', unlockWrite); + source.on('data', lockWrite); + + source.prependListener('end', unlockWrite); + source.on('end', lockWrite); + }); + + this.on('unpipe', (source: Writable) => { + source.off('data', unlockWrite); + source.off('data', lockWrite); + + source.off('end', unlockWrite); + source.off('end', lockWrite); + }); + + this.on('pipe', source => { + if (source instanceof IncomingMessage) { + this.options.headers = { + ...source.headers, + ...this.options.headers + }; + } + }); + + const {json, body, form} = options; + if (json || body || form) { + this._lockWrite(); + } + + (async (nonNormalizedOptions: Options) => { + try { + if (nonNormalizedOptions.body instanceof ReadStream) { + await waitForOpenFile(nonNormalizedOptions.body); + } + + if (kIsNormalizedAlready in nonNormalizedOptions) { + this.options = nonNormalizedOptions as NormalizedOptions; + } else { + // @ts-ignore Common TypeScript bug saying that `this.constructor` is not accessible + this.options = this.constructor.normalizeArguments(url, nonNormalizedOptions, defaults); + } + + const {url: normalizedURL} = this.options; + + if (!normalizedURL) { + throw new TypeError('Missing `url` property'); + } + + this.requestUrl = normalizedURL.toString(); + decodeURI(this.requestUrl); + + await this._finalizeBody(); + await this._makeRequest(); + + this.finalized = true; + this.emit('finalized'); + } catch (error) { + if (error instanceof RequestError) { + this._beforeError(error); + return; + } + + this.destroy(error); + } + })(options); + } + + static normalizeArguments(url?: string | URL, options?: Options, defaults?: Defaults): NormalizedOptions { + const rawOptions = options; + + if (is.object(url) && !is.urlInstance(url)) { + options = {...defaults as NormalizedOptions, ...(url as Options), ...options}; + } else { + if (url && options && options.url) { + throw new TypeError('The `url` option is mutually exclusive with the `input` argument'); + } + + options = {...defaults as NormalizedOptions, ...options}; + + if (url) { + options.url = url; + } + } + + if (rawOptions && defaults) { + for (const key in rawOptions) { + // @ts-ignore Dear TypeScript, all object keys are strings (or symbols which are NOT enumerable). + if (is.undefined(rawOptions[key]) && !is.undefined(defaults[key])) { + // @ts-ignore See the note above + options[key] = defaults[key]; + } + } + } + + // TODO: Deprecate URL options in Got 12. + + // Support extend-specific options + if (options.cache === false) { + options.cache = undefined; + } + + // Nice type assertions + assert.any([is.string, is.undefined], options.method); + assert.any([is.object, is.undefined], options.headers); + assert.any([is.string, is.urlInstance, is.undefined], options.prefixUrl); + assert.any([is.object, is.undefined], options.cookieJar); + assert.any([is.object, is.string, is.undefined], options.searchParams); + assert.any([is.object, is.string, is.undefined], options.cache); + assert.any([is.object, is.number, is.undefined], options.timeout); + assert.any([is.object, is.undefined], options.context); + assert.any([is.object, is.undefined], options.hooks); + assert.any([is.boolean, is.undefined], options.decompress); + assert.any([is.boolean, is.undefined], options.ignoreInvalidCookies); + assert.any([is.boolean, is.undefined], options.followRedirect); + assert.any([is.number, is.undefined], options.maxRedirects); + assert.any([is.boolean, is.undefined], options.throwHttpErrors); + assert.any([is.boolean, is.undefined], options.http2); + assert.any([is.boolean, is.undefined], options.allowGetBody); + assert.any([is.boolean, is.undefined], options.rejectUnauthorized); + + // `options.method` + if (is.string(options.method)) { + options.method = options.method.toUpperCase() as Method; + } else { + options.method = 'GET'; + } + + // `options.headers` + if (is.undefined(options.headers)) { + options.headers = {}; + } else { + options.headers = lowercaseKeys({...(defaults?.headers), ...options.headers}); + } + + // Disallow legacy `url.Url` + if ('slashes' in options) { + throw new TypeError('The legacy `url.Url` has been deprecated. Use `URL` instead.'); + } + + // `options.auth` + if ('auth' in options) { + throw new TypeError('Parameter `auth` is deprecated. Use `username` / `password` instead.'); + } + + // `options.prefixUrl` & `options.url` + if (options.prefixUrl) { + options.prefixUrl = options.prefixUrl.toString(); + + if (options.prefixUrl !== '' && !options.prefixUrl.endsWith('/')) { + options.prefixUrl += '/'; + } + } else { + options.prefixUrl = ''; + } + + if (is.string(options.url)) { + if (options.url.startsWith('/')) { + throw new Error('`input` must not start with a slash when using `prefixUrl`'); + } + + options.url = optionsToUrl(options.prefixUrl + options.url, options as Options & {searchParams?: URLSearchParams}); + } else if ((is.undefined(options.url) && options.prefixUrl !== '') || options.protocol) { + options.url = optionsToUrl(options.prefixUrl, options as Options & {searchParams?: URLSearchParams}); + } + + if (options.url) { + // Make it possible to change `options.prefixUrl` + let {prefixUrl} = options; + Object.defineProperty(options, 'prefixUrl', { + set: (value: string) => { + const url = options!.url as URL; + + if (!url.href.startsWith(value)) { + throw new Error(`Cannot change \`prefixUrl\` from ${prefixUrl} to ${value}: ${url.href}`); + } + + options!.url = new URL(value + url.href.slice(prefixUrl.length)); + prefixUrl = value; + }, + get: () => prefixUrl + }); + + // Protocol check + let {protocol} = options.url; + + if (protocol === 'unix:') { + protocol = 'http:'; + + options.url = new URL(`http://unix${options.url.pathname}${options.url.search}`); + } + + if (options.url.search) { + const triggerSearchParams = '_GOT_INTERNAL_TRIGGER_NORMALIZATION'; + + options.url.searchParams.append(triggerSearchParams, ''); + options.url.searchParams.delete(triggerSearchParams); + } + + if (protocol !== 'http:' && protocol !== 'https:') { + throw new UnsupportedProtocolError(options as NormalizedOptions); + } + } + + // `options.username` & `options.password` + options.username = options.username ?? ''; + options.password = options.password ?? ''; + + if (options.url) { + options.url.username = options.username; + options.url.password = options.password; + } + + // `options.cookieJar` + if (options.cookieJar) { + let {setCookie, getCookieString} = options.cookieJar; + + // Horrible `tough-cookie` check + if (setCookie.length === 4 && getCookieString.length === 0) { + setCookie = promisify(setCookie.bind(options.cookieJar)); + getCookieString = promisify(getCookieString.bind(options.cookieJar)); + } else if (setCookie.length !== 2) { + throw new TypeError('`options.cookieJar.setCookie` needs to be an async function with 2 arguments'); + } else if (getCookieString.length !== 1) { + throw new TypeError('`options.cookieJar.getCookieString` needs to be an async function with 1 argument'); + } + + options.cookieJar = {setCookie, getCookieString}; + } + + // `options.searchParams` + if (options.searchParams) { + if (!is.string(options.searchParams) && !(options.searchParams instanceof URLSearchParams)) { + validateSearchParams(options.searchParams); + } + + options.searchParams = new URLSearchParams(options.searchParams as Record); + + // `normalizeArguments()` is also used to merge options + const defaultsAsOptions = defaults as Options | undefined; + if (defaultsAsOptions && defaultsAsOptions.searchParams instanceof URLSearchParams) { + defaultsAsOptions.searchParams.forEach((value, key) => { + (options!.searchParams as URLSearchParams).append(key, value); + }); + } + + if (options.url) { + options.url.search = options.searchParams.toString(); + } + } + + // `options.cache` + if (options.cache && !(options as NormalizedOptions).cacheableRequest) { + // Better memory management, so we don't have to generate a new object every time + (options as NormalizedOptions).cacheableRequest = new CacheableRequest( + ((requestOptions: RequestOptions, handler?: (response: IncomingMessage) => void): ClientRequest => (requestOptions as Pick)[kRequest](requestOptions, handler)) as HttpRequestFunction, + options.cache + ); + } + + // `options.dnsCache` + if (options.dnsCache === true) { + options.dnsCache = new CacheableLookup(); + } else if (!is.undefined(options.dnsCache) && options.dnsCache !== false && !(options.dnsCache instanceof CacheableLookup)) { + throw new TypeError(`Parameter \`dnsCache\` must be a CacheableLookup instance or a boolean, got ${is(options.dnsCache)}`); + } + + // `options.timeout` + if (is.number(options.timeout)) { + options.timeout = {request: options.timeout}; + } else if (defaults) { + options.timeout = { + ...defaults.timeout, + ...options.timeout + }; + } else { + options.timeout = {...options.timeout}; + } + + // `options.context` + if (!options.context) { + options.context = {}; + } + + // `options.hooks` + options.hooks = {...options.hooks}; + + for (const event of knownHookEvents) { + if (event in options.hooks) { + if (is.array(options.hooks[event])) { + // See https://github.com/microsoft/TypeScript/issues/31445#issuecomment-576929044 + (options.hooks as any)[event] = [...options.hooks[event]!]; + } else { + throw new TypeError(`Parameter \`${event}\` must be an Array, got ${is(options.hooks[event])}`); + } + } else { + options.hooks[event] = []; + } + } + + if (defaults) { + for (const event of knownHookEvents) { + const defaultHooks = defaults.hooks[event]; + + if (defaultHooks.length !== 0) { + // See https://github.com/microsoft/TypeScript/issues/31445#issuecomment-576929044 + (options.hooks as any)[event] = [ + ...defaults.hooks[event], + ...options.hooks[event]! + ]; + } + } + } + + // Other options + if ('followRedirects' in options) { + throw new TypeError('The `followRedirects` option does not exist. Use `followRedirect` instead.'); + } + + if (options.agent) { + for (const key in options.agent) { + if (key !== 'http' && key !== 'https' && key !== 'http2') { + throw new TypeError(`Expected the \`options.agent\` properties to be \`http\`, \`https\` or \`http2\`, got \`${key}\``); + } + } + } + + options.maxRedirects = options.maxRedirects ?? 0; + + // Set non-enumerable properties + setNonEnumerableProperties([defaults, options], options); + + return options as NormalizedOptions; + } + + _lockWrite(): void { + const onLockedWrite = (): never => { + throw new TypeError('The payload has been already provided'); + }; + + this.write = onLockedWrite; + this.end = onLockedWrite; + } + + _unlockWrite(): void { + this.write = super.write; + this.end = super.end; + } + + async _finalizeBody(): Promise { + const {options} = this; + const {headers} = options; + + const isForm = !is.undefined(options.form); + const isJSON = !is.undefined(options.json); + const isBody = !is.undefined(options.body); + const hasPayload = isForm || isJSON || isBody; + const cannotHaveBody = withoutBody.has(options.method) && !(options.method === 'GET' && options.allowGetBody); + + this._cannotHaveBody = cannotHaveBody; + + if (hasPayload) { + if (cannotHaveBody) { + throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); + } + + if ([isBody, isForm, isJSON].filter(isTrue => isTrue).length > 1) { + throw new TypeError('The `body`, `json` and `form` options are mutually exclusive'); + } + + if ( + isBody && + !(options.body instanceof Readable) && + !is.string(options.body) && + !is.buffer(options.body) && + !isFormData(options.body) + ) { + throw new TypeError('The `body` option must be a stream.Readable, string or Buffer'); + } + + if (isForm && !is.object(options.form)) { + throw new TypeError('The `form` option must be an Object'); + } + + { + // Serialize body + const noContentType = !is.string(headers['content-type']); + + if (isBody) { + // Special case for https://github.com/form-data/form-data + if (isFormData(options.body) && noContentType) { + headers['content-type'] = `multipart/form-data; boundary=${options.body.getBoundary()}`; + } + } else if (isForm) { + if (noContentType) { + headers['content-type'] = 'application/x-www-form-urlencoded'; + } + + options.body = (new URLSearchParams(options.form as Record)).toString(); + } else { + if (noContentType) { + headers['content-type'] = 'application/json'; + } + + options.body = JSON.stringify(options.json); + } + + const uploadBodySize = await getBodySize(options); + + // See https://tools.ietf.org/html/rfc7230#section-3.3.2 + // A user agent SHOULD send a Content-Length in a request message when + // no Transfer-Encoding is sent and the request method defines a meaning + // for an enclosed payload body. For example, a Content-Length header + // field is normally sent in a POST request even when the value is 0 + // (indicating an empty payload body). A user agent SHOULD NOT send a + // Content-Length header field when the request message does not contain + // a payload body and the method semantics do not anticipate such a + // body. + if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding'])) { + if (!cannotHaveBody && !is.undefined(uploadBodySize)) { + headers['content-length'] = String(uploadBodySize); + } + } + } + } else if (cannotHaveBody) { + this._lockWrite(); + } else { + this._unlockWrite(); + } + + this[kBodySize] = Number(headers['content-length']) || undefined; + } + + async _onResponse(response: IncomingMessage): Promise { + const {options} = this; + const {url} = options; + + if (options.decompress) { + response = decompressResponse(response); + } + + const statusCode = response.statusCode!; + const typedResponse = response as Response; + + typedResponse.statusMessage = typedResponse.statusMessage === '' ? http.STATUS_CODES[statusCode] : typedResponse.statusMessage; + typedResponse.url = options.url.toString(); + typedResponse.requestUrl = this.requestUrl; + typedResponse.redirectUrls = this.redirects; + typedResponse.request = this; + typedResponse.isFromCache = (response as any).fromCache || false; + typedResponse.ip = this.ip; + + this[kIsFromCache] = typedResponse.isFromCache; + + this[kResponseSize] = Number(response.headers['content-length']) || undefined; + this[kResponse] = response; + + response.once('end', () => { + this[kResponseSize] = this[kDownloadedSize]; + this.emit('downloadProgress', this.downloadProgress); + }); + + response.on('error', (error: Error) => { + this._beforeError(new ReadError(error, options, response as Response)); + }); + + this.emit('downloadProgress', this.downloadProgress); + + const rawCookies = response.headers['set-cookie']; + if (is.object(options.cookieJar) && rawCookies) { + let promises: Array> = rawCookies.map(async (rawCookie: string) => (options.cookieJar as PromiseCookieJar).setCookie(rawCookie, url.toString())); + + if (options.ignoreInvalidCookies) { + promises = promises.map(async p => p.catch(() => {})); + } + + try { + await Promise.all(promises); + } catch (error) { + this._beforeError(error); + return; + } + } + + if (options.followRedirect && response.headers.location && redirectCodes.has(statusCode)) { + // We're being redirected, we don't care about the response. + // It'd be besto to abort the request, but we can't because + // we would have to sacrifice the TCP connection. We don't want that. + response.resume(); + + if (this[kRequest]) { + this[kCancelTimeouts]!(); + + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete this[kRequest]; + this[kUnproxyEvents](); + } + + const shouldBeGet = statusCode === 303 && options.method !== 'GET' && options.method !== 'HEAD'; + if (shouldBeGet || !options.methodRewriting) { + // Server responded with "see other", indicating that the resource exists at another location, + // and the client should request it from that location via GET or HEAD. + options.method = 'GET'; + + if ('body' in options) { + delete options.body; + } + + if ('json' in options) { + delete options.json; + } + + if ('form' in options) { + delete options.form; + } + } + + if (this.redirects.length >= options.maxRedirects) { + this._beforeError(new MaxRedirectsError(typedResponse, options.maxRedirects, options)); + return; + } + + try { + // Handles invalid URLs. See https://github.com/sindresorhus/got/issues/604 + const redirectBuffer = Buffer.from(response.headers.location, 'binary').toString(); + const redirectUrl = new URL(redirectBuffer, url); + const redirectString = redirectUrl.toString(); + decodeURI(redirectString); + + // Redirecting to a different site, clear sensitive data. + if (redirectUrl.hostname !== url.hostname) { + if ('cookie' in options.headers) { + delete options.headers.cookie; + } + + if ('authorization' in options.headers) { + delete options.headers.authorization; + } + + if (options.username || options.password) { + delete options.username; + delete options.password; + } + } + + this.redirects.push(redirectString); + options.url = redirectUrl; + + for (const hook of options.hooks.beforeRedirect) { + // eslint-disable-next-line no-await-in-loop + await hook(options, typedResponse); + } + + this.emit('redirect', typedResponse, options); + + await this._makeRequest(); + } catch (error) { + this._beforeError(error); + return; + } + + return; + } + + const limitStatusCode = options.followRedirect ? 299 : 399; + const isOk = (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304; + if (options.throwHttpErrors && !isOk) { + await this._beforeError(new HTTPError(typedResponse, options)); + + if (this.destroyed) { + return; + } + } + + // We need to call `_read()` only when the Request stream is flowing + response.on('readable', () => { + if ((this as any).readableFlowing) { + this._read(); + } + }); + + this.on('resume', () => { + response.resume(); + }); + + this.on('pause', () => { + response.pause(); + }); + + response.once('end', () => { + this.push(null); + }); + + this.emit('response', response); + + for (const destination of this[kServerResponsesPiped]) { + if (destination.headersSent) { + continue; + } + + // eslint-disable-next-line guard-for-in + for (const key in response.headers) { + const isAllowed = options.decompress ? key !== 'content-encoding' : true; + const value = response.headers[key]; + + if (isAllowed) { + destination.setHeader(key, value!); + } + } + + destination.statusCode = statusCode; + } + } + + _onRequest(request: ClientRequest): void { + const {options} = this; + const {timeout, url} = options; + + timer(request); + + this[kCancelTimeouts] = timedOut(request, timeout, url); + + request.once('response', response => { + this._onResponse(response); + }); + + request.once('error', (error: Error) => { + if (error instanceof TimedOutTimeoutError) { + error = new TimeoutError(error, this.timings!, options); + } else { + error = new RequestError(error.message, error, options, this); + } + + this._beforeError(error as RequestError); + }); + + this[kUnproxyEvents] = proxyEvents(request, this, proxiedRequestEvents); + + this[kRequest] = request; + + this.emit('uploadProgress', this.uploadProgress); + + // Send body + const currentRequest = this.redirects.length === 0 ? this : request; + if (is.nodeStream(options.body)) { + options.body.pipe(currentRequest); + options.body.once('error', (error: NodeJS.ErrnoException) => { + this._beforeError(new UploadError(error, options, this)); + }); + + options.body.once('end', () => { + delete options.body; + }); + } else { + this._unlockWrite(); + + if (!is.undefined(options.body)) { + this._writeRequest(options.body, null as unknown as string, () => {}); + currentRequest.end(); + + this._lockWrite(); + } else if (this._cannotHaveBody || this._noPipe) { + currentRequest.end(); + + this._lockWrite(); + } + } + + this.emit('request', request); + } + + async _makeRequest(): Promise { + const {options} = this; + const {url, headers, request, agent, timeout} = options; + + for (const key in headers) { + if (is.undefined(headers[key])) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete headers[key]; + } else if (is.null_(headers[key])) { + throw new TypeError(`Use \`undefined\` instead of \`null\` to delete the \`${key}\` header`); + } + } + + if (options.decompress && is.undefined(headers['accept-encoding'])) { + headers['accept-encoding'] = supportsBrotli ? 'gzip, deflate, br' : 'gzip, deflate'; + } + + // Set cookies + if (options.cookieJar) { + const cookieString: string = await options.cookieJar.getCookieString(options.url.toString()); + + if (is.nonEmptyString(cookieString)) { + options.headers.cookie = cookieString; + } + } + + for (const hook of options.hooks.beforeRequest) { + // eslint-disable-next-line no-await-in-loop + const result = await hook(options); + + if (!is.undefined(result)) { + // @ts-ignore Skip the type mismatch to support abstract responses + options.request = () => result; + break; + } + } + + if (options.dnsCache && !('lookup' in options)) { + options.lookup = options.dnsCache.lookup; + } + + // UNIX sockets + if (url.hostname === 'unix') { + const matches = /(?.+?):(?.+)/.exec(`${url.pathname}${url.search}`); + + if (matches?.groups) { + const {socketPath, path} = matches.groups; + + Object.assign(options, { + socketPath, + path, + host: '' + }); + } + } + + const isHttps = url.protocol === 'https:'; + + let fallbackFn: HttpRequestFunction; + if (options.http2) { + fallbackFn = http2wrapper.auto; + } else { + fallbackFn = isHttps ? https.request : http.request; + } + + const realFn = options.request ?? fallbackFn; + const fn = options.cacheableRequest ? cacheFn : realFn; + + if (agent && !options.http2) { + (options as unknown as RequestOptions).agent = agent[isHttps ? 'https' : 'http']; + } + + options[kRequest] = realFn as HttpRequestFunction; + delete options.request; + delete options.timeout; + + let requestOrResponse: ReturnType; + + try { + requestOrResponse = await fn(url, options as unknown as RequestOptions); + + if (is.undefined(requestOrResponse)) { + requestOrResponse = fallbackFn(url, options as unknown as RequestOptions); + } + + // Restore options + options.request = request; + options.timeout = timeout; + options.agent = agent; + + if (isClientRequest(requestOrResponse)) { + this._onRequest(requestOrResponse); + + // Emit the response after the stream has been ended + } else if (this.writable) { + this.once('finish', () => { + this._onResponse(requestOrResponse as IncomingMessage); + }); + + this._unlockWrite(); + this.end(); + this._lockWrite(); + } else { + this._onResponse(requestOrResponse as IncomingMessage); + } + } catch (error) { + if (error instanceof RequestError) { + throw error; + } + + throw new RequestError(error.message, error, options, this); + } + } + + async _beforeError(error: Error): Promise { + this.errored = true; + + if (!(error instanceof RequestError)) { + error = new RequestError(error.message, error, this.options, this); + } + + try { + const {response} = error as RequestError; + if (response && is.undefined(response.body)) { + response.body = await getStream(response, { + ...this.options, + encoding: (this as any)._readableState.encoding + }); + } + } catch (_) {} + + try { + for (const hook of this.options.hooks.beforeError) { + // eslint-disable-next-line no-await-in-loop + error = await hook(error as RequestError); + } + } catch (error_) { + error = new RequestError(error_.message, error_, this.options, this); + } + + this.destroy(error); + } + + _read(): void { + if (kResponse in this && !this.errored) { + let data; + + while ((data = this[kResponse]!.read()) !== null) { + this[kDownloadedSize] += data.length; + this[kStartedReading] = true; + + const progress = this.downloadProgress; + + if (progress.percent < 1) { + this.emit('downloadProgress', progress); + } + + this.push(data); + } + } + } + + _write(chunk: any, encoding: string, callback: (error?: Error | null) => void): void { + const write = (): void => { + this._writeRequest(chunk, encoding, callback); + }; + + if (this.finalized) { + write(); + } else { + this.once('finalized', write); + } + } + + _writeRequest(chunk: any, encoding: string, callback: (error?: Error | null) => void): void { + this._progressCallbacks.push((): void => { + this[kUploadedSize] += Buffer.byteLength(chunk, encoding as BufferEncoding); + + const progress = this.uploadProgress; + + if (progress.percent < 1) { + this.emit('uploadProgress', progress); + } + }); + + this[kRequest]!.write(chunk, encoding, (error?: Error | null) => { + if (!error && this._progressCallbacks.length !== 0) { + this._progressCallbacks.shift()!(); + } + + callback(error); + }); + } + + _final(callback: (error?: Error | null) => void): void { + const endRequest = (): void => { + // FIX: Node.js 10 calls the write callback AFTER the end callback! + while (this._progressCallbacks.length !== 0) { + this._progressCallbacks.shift()!(); + } + + // We need to check if `this[kRequest]` is present, + // because it isn't when we use cache. + if (!(kRequest in this)) { + callback(); + return; + } + + this[kRequest]!.end((error?: Error | null) => { + if (!error) { + this[kBodySize] = this[kUploadedSize]; + + this.emit('uploadProgress', this.uploadProgress); + this[kRequest]!.emit('upload-complete'); + } + + callback(error); + }); + }; + + if (this.finalized) { + endRequest(); + } else { + this.once('finalized', endRequest); + } + } + + _destroy(error: Error | null, callback: (error: Error | null) => void): void { + if (kRequest in this) { + this[kRequest]!.abort(); + } else { + this.once('finalized', (): void => { + if (kRequest in this) { + this[kRequest]!.abort(); + } + }); + } + + if (error !== null && !is.undefined(error) && !(error instanceof RequestError)) { + error = new RequestError(error.message, error, this.options, this); + } + + callback(error); + } + + get ip(): string | undefined { + return this[kRequest]?.socket.remoteAddress; + } + + get aborted(): boolean { + return Boolean(this[kRequest]?.aborted); + } + + get downloadProgress(): Progress { + let percent; + if (this[kResponseSize]) { + percent = this[kDownloadedSize] / this[kResponseSize]!; + } else if (this[kResponseSize] === this[kDownloadedSize]) { + percent = 1; + } else { + percent = 0; + } + + return { + percent, + transferred: this[kDownloadedSize], + total: this[kResponseSize] + }; + } + + get uploadProgress(): Progress { + let percent; + if (this[kBodySize]) { + percent = this[kUploadedSize] / this[kBodySize]!; + } else if (this[kBodySize] === this[kUploadedSize]) { + percent = 1; + } else { + percent = 0; + } + + return { + percent, + transferred: this[kUploadedSize], + total: this[kBodySize] + }; + } + + get timings(): Timings | undefined { + return (this[kRequest] as ClientRequestWithTimings)?.timings; + } + + get isFromCache(): boolean | undefined { + return this[kIsFromCache]; + } + + pipe(destination: T, options?: {end?: boolean}): T { + if (this[kStartedReading]) { + throw new Error('Failed to pipe. The response has been emitted already.'); + } + + if (destination instanceof ServerResponse) { + this[kServerResponsesPiped].add(destination); + } + + return super.pipe(destination, options); + } + + unpipe(destination: T): this { + if (destination instanceof ServerResponse) { + this[kServerResponsesPiped].delete(destination); + } + + super.unpipe(destination); + + return this; + } +} diff --git a/source/utils/get-body-size.ts b/source/core/utils/get-body-size.ts similarity index 100% rename from source/utils/get-body-size.ts rename to source/core/utils/get-body-size.ts diff --git a/source/core/utils/is-form-data.ts b/source/core/utils/is-form-data.ts new file mode 100644 index 000000000..cd368fbc9 --- /dev/null +++ b/source/core/utils/is-form-data.ts @@ -0,0 +1,9 @@ +import is from '@sindresorhus/is'; +import {Readable} from 'stream'; + +interface FormData extends Readable { + getBoundary(): string; + getLength(callback: (error: Error | null, length: number) => void): void; +} + +export default (body: unknown): body is FormData => is.nodeStream(body) && is.function_((body as FormData).getBoundary); diff --git a/source/core/utils/options-to-url.ts b/source/core/utils/options-to-url.ts new file mode 100644 index 000000000..0f00e9805 --- /dev/null +++ b/source/core/utils/options-to-url.ts @@ -0,0 +1,73 @@ +/* istanbul ignore file: deprecated */ +import {URL} from 'url'; + +export interface URLOptions { + href?: string; + protocol?: string; + host?: string; + hostname?: string; + port?: string | number; + pathname?: string; + search?: string; + searchParams?: unknown; + path?: string; +} + +const keys: Array> = [ + 'protocol', + 'host', + 'hostname', + 'port', + 'pathname', + 'search' +]; + +export default (origin: string, options: URLOptions): URL => { + if (options.path) { + if (options.pathname) { + throw new TypeError('Parameters `path` and `pathname` are mutually exclusive.'); + } + + if (options.search) { + throw new TypeError('Parameters `path` and `search` are mutually exclusive.'); + } + + if (options.searchParams) { + throw new TypeError('Parameters `path` and `searchParams` are mutually exclusive.'); + } + } + + if (options.search && options.searchParams) { + throw new TypeError('Parameters `search` and `searchParams` are mutually exclusive.'); + } + + if (!origin) { + if (!options.protocol) { + throw new TypeError('No URL protocol specified'); + } + + origin = `${options.protocol}//${options.hostname ?? options.host ?? ''}`; + } + + const url = new URL(origin); + + if (options.path) { + const searchIndex = options.path.indexOf('?'); + if (searchIndex === -1) { + options.pathname = options.path; + } else { + options.pathname = options.path.slice(0, searchIndex); + options.search = options.path.slice(searchIndex + 1); + } + + delete options.path; + } + + for (const key of keys) { + if (options[key]) { + url[key] = options[key]!.toString(); + } + } + + return url; +}; diff --git a/source/core/utils/proxy-events.ts b/source/core/utils/proxy-events.ts new file mode 100644 index 000000000..bd78b4a64 --- /dev/null +++ b/source/core/utils/proxy-events.ts @@ -0,0 +1,22 @@ +import {EventEmitter} from 'events'; + +type Fn = (...args: unknown[]) => void; +type Fns = {[key: string]: Fn}; + +export default function (from: EventEmitter, to: EventEmitter, events: string[]): () => void { + const fns: Fns = {}; + + for (const event of events) { + fns[event] = (...args: unknown[]) => { + to.emit(event, ...args); + }; + + from.on(event, fns[event]); + } + + return () => { + for (const event of events) { + from.off(event, fns[event]); + } + }; +} diff --git a/source/utils/timed-out.ts b/source/core/utils/timed-out.ts similarity index 93% rename from source/utils/timed-out.ts rename to source/core/utils/timed-out.ts index 55b7b98f6..57cd75e49 100644 --- a/source/utils/timed-out.ts +++ b/source/core/utils/timed-out.ts @@ -43,7 +43,7 @@ export class TimeoutError extends Error { } export default (request: ClientRequest, delays: Delays, options: TimedOutOptions): () => void => { - if (Reflect.has(request, reentry)) { + if (reentry in request) { return noop; } @@ -69,12 +69,10 @@ export default (request: ClientRequest, delays: Delays, options: TimedOutOptions const timeoutHandler = (delay: number, event: string): void => { if (request.socket) { - // @ts-ignore We do not want the `socket hang up` error - request.socket._hadError = true; + (request.socket as any)._hadError = true; } - request.abort(); - request.emit('error', new TimeoutError(delay, event)); + request.destroy(new TimeoutError(delay, event)); }; const cancelTimeouts = (): void => { @@ -120,8 +118,7 @@ export default (request: ClientRequest, delays: Delays, options: TimedOutOptions } once(request, 'socket', (socket: net.Socket): void => { - // @ts-ignore Node typings doesn't have this property - const {socketPath} = request; + const {socketPath} = request as ClientRequest & {socketPath?: string}; /* istanbul ignore next: hard to test */ if (socket.connecting) { diff --git a/source/utils/unhandle.ts b/source/core/utils/unhandle.ts similarity index 95% rename from source/utils/unhandle.ts rename to source/core/utils/unhandle.ts index 723aa9f19..7f0a838ba 100644 --- a/source/utils/unhandle.ts +++ b/source/core/utils/unhandle.ts @@ -1,4 +1,4 @@ -import EventEmitter = require('events'); +import {EventEmitter} from 'events'; type Origin = EventEmitter; type Event = string | symbol; diff --git a/source/utils/url-to-options.ts b/source/core/utils/url-to-options.ts similarity index 100% rename from source/utils/url-to-options.ts rename to source/core/utils/url-to-options.ts diff --git a/source/create.ts b/source/create.ts index 43acfa04a..08e2f922a 100644 --- a/source/create.ts +++ b/source/create.ts @@ -1,98 +1,66 @@ -import {Merge, Except} from 'type-fest'; -import is from '@sindresorhus/is'; -import asPromise, {createRejection} from './as-promise'; -import asStream, {ProxyStream} from './as-stream'; -import * as errors from './errors'; -import {normalizeArguments, mergeOptions} from './normalize-arguments'; -import deepFreeze from './utils/deep-freeze'; +import {URL} from 'url'; +import {CancelError} from 'p-cancelable'; +import is from '@sindresorhus/is/dist'; +import asPromise, { + // Request & Response + PromisableRequest, + Response, + + // Options + Options, + NormalizedOptions, + + // Hooks + InitHook, + + // Errors + ParseError, + RequestError, + CacheError, + ReadError, + HTTPError, + MaxRedirectsError, + TimeoutError, + UnsupportedProtocolError, + UploadError +} from './as-promise'; import { - CancelableRequest, - Defaults, - DefaultOptions, + GotReturn, ExtendOptions, + Got, + HTTPAlias, HandlerFunction, - NormalizedOptions, - Options, - Response, - URLOrOptions, - PaginationOptions + InstanceDefaults, + GotPaginate, + GotStream, + GotRequestFunction, + OptionsWithPagination } from './types'; +import createRejection from './as-promise/create-rejection'; +import Request, {kIsNormalizedAlready} from './core'; +import deepFreeze from './utils/deep-freeze'; -export type HTTPAlias = - | 'get' - | 'post' - | 'put' - | 'patch' - | 'head' - | 'delete'; +const errors = { + RequestError, + CacheError, + ReadError, + HTTPError, + MaxRedirectsError, + TimeoutError, + ParseError, + CancelError, + UnsupportedProtocolError, + UploadError +}; -export type ReturnStream = (url: string | Merge, options?: Merge) => ProxyStream; -export type GotReturn = CancelableRequest | ProxyStream; +const {normalizeArguments, mergeOptions} = PromisableRequest; -const getPromiseOrStream = (options: NormalizedOptions): GotReturn => options.isStream ? asStream(options) : asPromise(options); +const getPromiseOrStream = (options: NormalizedOptions): GotReturn => options.isStream ? new Request(options.url, options) : asPromise(options); const isGotInstance = (value: Got | ExtendOptions): value is Got => ( - Reflect.has(value, 'defaults') && Reflect.has(value.defaults, 'options') + 'defaults' in value && 'options' in value.defaults ); -export type OptionsOfDefaultResponseBody = Merge; -type OptionsOfTextResponseBody = Merge; -type OptionsOfJSONResponseBody = Merge; -type OptionsOfBufferResponseBody = Merge; -type ResponseBodyOnly = {resolveBodyOnly: true}; - -/** -Can be used to match methods explicitly or parameters extraction: `Parameters`. -*/ -export interface GotRequestMethod { - // `asPromise` usage - (url: string | OptionsOfDefaultResponseBody, options?: OptionsOfDefaultResponseBody): CancelableRequest>; - (url: string | OptionsOfTextResponseBody, options?: OptionsOfTextResponseBody): CancelableRequest>; - (url: string | OptionsOfJSONResponseBody, options?: OptionsOfJSONResponseBody): CancelableRequest>; - (url: string | OptionsOfBufferResponseBody, options?: OptionsOfBufferResponseBody): CancelableRequest>; - - // `resolveBodyOnly` usage - (url: string | Merge, options?: Merge): CancelableRequest; - (url: string | Merge, options?: Merge): CancelableRequest; - (url: string | Merge, options?: Merge): CancelableRequest; - (url: string | Merge, options?: Merge): CancelableRequest; - - // `asStream` usage - (url: string | Merge, options?: Merge): ProxyStream; -} - -export type GotPaginateOptions = Except> & PaginationOptions; -export type URLOrGotPaginateOptions = string | GotPaginateOptions; - -export interface GotPaginate { - (url: URLOrGotPaginateOptions, options?: GotPaginateOptions): AsyncIterableIterator; - all(url: URLOrGotPaginateOptions, options?: GotPaginateOptions): Promise; -} - -export interface Got extends Record, GotRequestMethod { - stream: GotStream; - paginate: GotPaginate; - defaults: Defaults; - GotError: typeof errors.GotError; - CacheError: typeof errors.CacheError; - RequestError: typeof errors.RequestError; - ReadError: typeof errors.ReadError; - ParseError: typeof errors.ParseError; - HTTPError: typeof errors.HTTPError; - MaxRedirectsError: typeof errors.MaxRedirectsError; - UnsupportedProtocolError: typeof errors.UnsupportedProtocolError; - TimeoutError: typeof errors.TimeoutError; - CancelError: typeof errors.CancelError; - - extend(...instancesOrOptions: Array): Got; - mergeInstances(parent: Got, ...instances: Got[]): Got; - mergeOptions(...sources: Options[]): NormalizedOptions; -} - -export interface GotStream extends Record { - (url: URLOrOptions, options?: Options): ProxyStream; -} - const aliases: readonly HTTPAlias[] = [ 'get', 'post', @@ -104,7 +72,15 @@ const aliases: readonly HTTPAlias[] = [ export const defaultHandler: HandlerFunction = (options, next) => next(options); -const create = (defaults: Defaults): Got => { +const callInitHooks = (hooks: InitHook[] | undefined, options: Options): void => { + if (hooks) { + for (const hook of hooks) { + hook(options); + } + } +}; + +const create = (defaults: InstanceDefaults): Got => { // Proxy properties from next handlers defaults._rawHandlers = defaults.handlers; defaults.handlers = defaults.handlers.map(fn => ((options, next) => { @@ -133,8 +109,7 @@ const create = (defaults: Defaults): Got => { return result; })); - // @ts-ignore Because the for loop handles it for us, as well as the other Object.defines - const got: Got = (url: URLOrOptions, options?: Options): GotReturn => { + const got: Got = ((url: string | URL, options: Options = {}): GotReturn => { let iteration = 0; const iterateHandlers = (newOptions: NormalizedOptions): GotReturn => { return defaults.handlers[iteration++]( @@ -143,19 +118,46 @@ const create = (defaults: Defaults): Got => { ) as GotReturn; }; - /* eslint-disable @typescript-eslint/return-await */ + if (is.plainObject(url)) { + options = { + ...url as Options, + ...options + }; + + url = undefined as any; + } + try { - return iterateHandlers(normalizeArguments(url, options, defaults)); + // Call `init` hooks + let initHookError: Error | undefined; + try { + callInitHooks(defaults.options.hooks.init, options); + callInitHooks(options?.hooks?.init, options); + } catch (error) { + initHookError = error; + } + + // Normalize options & call handlers + const normalizedOptions = normalizeArguments(url, options, defaults.options); + normalizedOptions[kIsNormalizedAlready] = true; + + if (initHookError) { + throw new RequestError(initHookError.message, initHookError, normalizedOptions); + } + + // A bug. + // eslint-disable-next-line @typescript-eslint/return-await + return iterateHandlers(normalizedOptions); } catch (error) { if (options?.isStream) { throw error; } else { - // @ts-ignore It's an Error not a response, but TS thinks it's calling .resolve - return createRejection(error); + // A bug. + // eslint-disable-next-line @typescript-eslint/return-await + return createRejection(error, defaults.options.hooks.beforeError, options?.hooks?.beforeError); } } - /* eslint-enable @typescript-eslint/return-await */ - }; + }) as Got; got.extend = (...instancesOrOptions) => { const optionsArray: Options[] = [defaults.options]; @@ -170,8 +172,8 @@ const create = (defaults: Defaults): Got => { } else { optionsArray.push(value); - if (Reflect.has(value, 'handlers')) { - handlers.push(...value.handlers); + if ('handlers' in value) { + handlers.push(...value.handlers!); } isMutableDefaults = value.mutableDefaults; @@ -185,45 +187,36 @@ const create = (defaults: Defaults): Got => { } return create({ - options: mergeOptions(...optionsArray) as DefaultOptions, + options: mergeOptions(...optionsArray), handlers, mutableDefaults: Boolean(isMutableDefaults) }); }; - // @ts-ignore The missing methods because the for-loop handles it for us - got.stream = (url, options) => got(url, {...options, isStream: true}); - - for (const method of aliases) { - // @ts-ignore Cannot properly type a function with multiple definitions yet - got[method] = (url: URLOrOptions, options?: Options): GotReturn => got(url, {...options, method}); - got.stream[method] = (url, options) => got.stream(url, {...options, method}); - } - - // @ts-ignore The missing property is added below - got.paginate = async function * (url: URLOrGotPaginateOptions, options?: GotPaginateOptions) { - let normalizedOptions = normalizeArguments(url as URLOrOptions, options as Options, defaults); + got.paginate = (async function * (url: string | URL, options?: OptionsWithPagination) { + let normalizedOptions = normalizeArguments(url, options, defaults.options); + normalizedOptions.resolveBodyOnly = false; - const pagination = normalizedOptions._pagination!; + const pagination = normalizedOptions.pagination!; if (!is.object(pagination)) { - throw new Error('`options._pagination` must be implemented'); + throw new TypeError('`options.pagination` must be implemented'); } const all: T[] = []; while (true) { - // @ts-ignore See https://github.com/sindresorhus/got/issues/954 + // TODO: Throw when result is not an instance of Response // eslint-disable-next-line no-await-in-loop - const result = await got(normalizedOptions); + const result = (await got('', normalizedOptions)) as Response; // eslint-disable-next-line no-await-in-loop - const parsed = await pagination.transform!(result); + const parsed = await pagination.transform(result); const current: T[] = []; for (const item of parsed) { - if (pagination.filter!(item, all, current)) { - if (!pagination.shouldContinue!(item, all, current)) { + if (pagination.filter(item, all, current)) { + if (!pagination.shouldContinue(item, all, current)) { return; } @@ -238,19 +231,19 @@ const create = (defaults: Defaults): Got => { } } - const optionsToMerge = pagination.paginate!(result, all, current); + const optionsToMerge = pagination.paginate(result, all, current); if (optionsToMerge === false) { return; } if (optionsToMerge !== undefined) { - normalizedOptions = normalizeArguments(normalizedOptions, optionsToMerge); + normalizedOptions = normalizeArguments(undefined, optionsToMerge, normalizedOptions); } } - }; + }) as GotPaginate; - got.paginate.all = async (url: URLOrGotPaginateOptions, options?: GotPaginateOptions) => { + got.paginate.all = (async (url: string | URL, options?: OptionsWithPagination) => { const results: T[] = []; for await (const item of got.paginate(url, options)) { @@ -258,7 +251,17 @@ const create = (defaults: Defaults): Got => { } return results; - }; + }) as GotPaginate['all']; + + got.stream = ((url: string | URL, options?: Options) => got(url, {...options, isStream: true})) as GotStream; + + for (const method of aliases) { + got[method] = ((url: string | URL, options?: Options): GotReturn => got(url, {...options, method})) as GotRequestFunction; + + got.stream[method] = ((url: string | URL, options?: Options & {isStream: true}) => { + return got(url, {...options, method, isStream: true}); + }) as GotStream; + } Object.assign(got, {...errors, mergeOptions}); Object.defineProperty(got, 'defaults', { @@ -272,3 +275,4 @@ const create = (defaults: Defaults): Got => { }; export default create; +export * from './types'; diff --git a/source/errors.ts b/source/errors.ts deleted file mode 100644 index ec62394a0..000000000 --- a/source/errors.ts +++ /dev/null @@ -1,125 +0,0 @@ -import is from '@sindresorhus/is'; -import {Timings} from '@szmarczak/http-timer'; -import {TimeoutError as TimedOutError} from './utils/timed-out'; -import {Response, NormalizedOptions} from './types'; - -export class GotError extends Error { - code?: string; - stack!: string; - declare readonly options: NormalizedOptions; - - constructor(message: string, error: Partial, options: NormalizedOptions) { - super(message); - Error.captureStackTrace(this, this.constructor); - this.name = 'GotError'; - - if (!is.undefined(error.code)) { - this.code = error.code; - } - - Object.defineProperty(this, 'options', { - // This fails because of TS 3.7.2 useDefineForClassFields - // Ref: https://github.com/microsoft/TypeScript/issues/34972 - enumerable: false, - value: options - }); - - // Recover the original stacktrace - if (!is.undefined(error.stack)) { - const indexOfMessage = this.stack.indexOf(this.message) + this.message.length; - const thisStackTrace = this.stack.slice(indexOfMessage).split('\n').reverse(); - const errorStackTrace = error.stack.slice(error.stack.indexOf(error.message!) + error.message!.length).split('\n').reverse(); - - // Remove duplicated traces - while (errorStackTrace.length !== 0 && errorStackTrace[0] === thisStackTrace[0]) { - thisStackTrace.shift(); - } - - this.stack = `${this.stack.slice(0, indexOfMessage)}${thisStackTrace.reverse().join('\n')}${errorStackTrace.reverse().join('\n')}`; - } - } -} - -export class CacheError extends GotError { - constructor(error: Error, options: NormalizedOptions) { - super(error.message, error, options); - this.name = 'CacheError'; - } -} - -export class RequestError extends GotError { - constructor(error: Error, options: NormalizedOptions) { - super(error.message, error, options); - this.name = 'RequestError'; - } -} - -export class ReadError extends GotError { - constructor(error: Error, options: NormalizedOptions) { - super(error.message, error, options); - this.name = 'ReadError'; - } -} - -export class ParseError extends GotError { - declare readonly response: Response; - - constructor(error: Error, response: Response, options: NormalizedOptions) { - super(`${error.message} in "${options.url.toString()}"`, error, options); - this.name = 'ParseError'; - - Object.defineProperty(this, 'response', { - enumerable: false, - value: response - }); - } -} - -export class HTTPError extends GotError { - declare readonly response: Response; - - constructor(response: Response, options: NormalizedOptions) { - super(`Response code ${response.statusCode} (${response.statusMessage!})`, {}, options); - this.name = 'HTTPError'; - - Object.defineProperty(this, 'response', { - enumerable: false, - value: response - }); - } -} - -export class MaxRedirectsError extends GotError { - declare readonly response: Response; - - constructor(response: Response, maxRedirects: number, options: NormalizedOptions) { - super(`Redirected ${maxRedirects} times. Aborting.`, {}, options); - this.name = 'MaxRedirectsError'; - - Object.defineProperty(this, 'response', { - enumerable: false, - value: response - }); - } -} - -export class UnsupportedProtocolError extends GotError { - constructor(options: NormalizedOptions) { - super(`Unsupported protocol "${options.url.protocol}"`, {}, options); - this.name = 'UnsupportedProtocolError'; - } -} - -export class TimeoutError extends GotError { - timings: Timings; - event: string; - - constructor(error: TimedOutError, timings: Timings, options: NormalizedOptions) { - super(error.message, error, options); - this.name = 'TimeoutError'; - this.event = error.event; - this.timings = timings; - } -} - -export {CancelError} from 'p-cancelable'; diff --git a/source/get-response.ts b/source/get-response.ts deleted file mode 100644 index bc1ceb394..000000000 --- a/source/get-response.ts +++ /dev/null @@ -1,37 +0,0 @@ -import decompressResponse = require('decompress-response'); -import EventEmitter = require('events'); -import mimicResponse = require('mimic-response'); -import stream = require('stream'); -import {IncomingMessage} from 'http'; -import {promisify} from 'util'; -import {createProgressStream} from './progress'; -import {NormalizedOptions} from './types'; - -const pipeline = promisify(stream.pipeline); - -export default async (response: IncomingMessage, options: NormalizedOptions, emitter: EventEmitter): Promise => { - const downloadBodySize = Number(response.headers['content-length']) || undefined; - const progressStream = createProgressStream('downloadProgress', emitter, downloadBodySize); - - mimicResponse(response, progressStream); - - const newResponse = ( - options.decompress && - options.method !== 'HEAD' ? decompressResponse(progressStream as unknown as IncomingMessage) : progressStream - ) as IncomingMessage; - - if (!options.decompress && ['gzip', 'deflate', 'br'].includes(newResponse.headers['content-encoding'] ?? '')) { - options.responseType = 'buffer'; - } - - emitter.emit('response', newResponse); - - return pipeline( - response, - progressStream - ).catch(error => { - if (error.code !== 'ERR_STREAM_PREMATURE_CLOSE') { - throw error; - } - }); -}; diff --git a/source/index.ts b/source/index.ts index 21882cdeb..a05d358ff 100644 --- a/source/index.ts +++ b/source/index.ts @@ -1,8 +1,9 @@ import {URL} from 'url'; -import create, {defaultHandler} from './create'; -import {Defaults, Response, GotOptions} from './types'; +import CacheableLookup from 'cacheable-lookup'; +import {Response, Options} from './as-promise'; +import create, {defaultHandler, InstanceDefaults} from './create'; -const defaults: Defaults = { +const defaults: InstanceDefaults = { options: { method: 'GET', retry: { @@ -52,22 +53,24 @@ const defaults: Defaults = { beforeError: [], afterResponse: [] }, + cache: undefined, + dnsCache: new CacheableLookup(), decompress: true, throwHttpErrors: true, followRedirect: true, isStream: false, - cache: false, - dnsCache: false, - useElectronNet: false, responseType: 'text', resolveBodyOnly: false, maxRedirects: 10, prefixUrl: '', methodRewriting: true, - allowGetBody: false, ignoreInvalidCookies: false, context: {}, - _pagination: { + // TODO: Set this to `true` when Got 12 gets released + http2: false, + allowGetBody: false, + rejectUnauthorized: true, + pagination: { transform: (response: Response) => { if (response.request.options.responseType === 'json') { return response.body; @@ -94,7 +97,7 @@ const defaults: Defaults = { } if (next) { - const options: GotOptions = { + const options: Options = { url: new URL(next) }; @@ -120,42 +123,5 @@ export default got; module.exports = got; module.exports.default = got; -// Export types -export * from './types'; - -export { - Got, - GotStream, - ReturnStream, - GotRequestMethod, - GotReturn -} from './create'; - -export { - ProxyStream as ResponseStream -} from './as-stream'; - -export { - GotError, - CacheError, - RequestError, - ReadError, - ParseError, - HTTPError, - MaxRedirectsError, - UnsupportedProtocolError, - TimeoutError, - CancelError -} from './errors'; - -export { - InitHook, - BeforeRequestHook, - BeforeRedirectHook, - BeforeRetryHook, - BeforeErrorHook, - AfterResponseHook, - HookType, - Hooks, - HookEvent -} from './known-hook-events'; +export * from './create'; +export * from './as-promise'; diff --git a/source/known-hook-events.ts b/source/known-hook-events.ts deleted file mode 100644 index 08120857b..000000000 --- a/source/known-hook-events.ts +++ /dev/null @@ -1,117 +0,0 @@ -import {CancelableRequest, GeneralError, NormalizedOptions, Options, Response} from './types'; - -/** -Called with plain request options, right before their normalization. This is especially useful in conjunction with `got.extend()` when the input needs custom handling. - -**Note:** This hook must be synchronous. - -@see [Request migration guide](https://github.com/sindresorhus/got/blob/master/migration-guides.md#breaking-changes) for an example. -*/ -export type InitHook = (options: Options) => void; - -/** -Called with normalized [request options](https://github.com/sindresorhus/got#options). Got will make no further changes to the request before it is sent (except the body serialization). This is especially useful in conjunction with [`got.extend()`](https://github.com/sindresorhus/got#instances) when you want to create an API client that, for example, uses HMAC-signing. - -@see [AWS section](https://github.com/sindresorhus/got#aws) for an example. -*/ -export type BeforeRequestHook = (options: NormalizedOptions) => void | Promise; - -/** -Called with normalized [request options](https://github.com/sindresorhus/got#options). Got will make no further changes to the request. This is especially useful when you want to avoid dead sites. -*/ -export type BeforeRedirectHook = (options: NormalizedOptions, response: Response) => void | Promise; - -/** -Called with normalized [request options](https://github.com/sindresorhus/got#options), the error and the retry count. Got will make no further changes to the request. This is especially useful when some extra work is required before the next try. -*/ -export type BeforeRetryHook = (options: NormalizedOptions, error?: GeneralError, retryCount?: number) => void | Promise; - -/** -Called with an `Error` instance. The error is passed to the hook right before it's thrown. This is especially useful when you want to have more detailed errors. - -**Note:** Errors thrown while normalizing input options are thrown directly and not part of this hook. -*/ -export type BeforeErrorHook = (error: ErrorLike) => GeneralError | Promise; - -/** -Called with [response object](https://github.com/sindresorhus/got#response) and a retry function. - -Each function should return the response. This is especially useful when you want to refresh an access token. -*/ -export type AfterResponseHook = (response: Response, retryWithMergedOptions: (options: Options) => CancelableRequest) => Response | CancelableRequest | Promise>; - -export type HookType = - | BeforeErrorHook - | InitHook - | BeforeRequestHook - | BeforeRedirectHook - | BeforeRetryHook - | AfterResponseHook; - -/** -Hooks allow modifications during the request lifecycle. Hook functions may be async and are run serially. -*/ -export interface Hooks { - /** - Called with plain request options, right before their normalization. This is especially useful in conjunction with `got.extend()` when the input needs custom handling. - - **Note:** This hook must be synchronous. - - @see [Request migration guide](https://github.com/sindresorhus/got/blob/master/migration-guides.md#breaking-changes) for an example. - @default [] - */ - init?: InitHook[]; - - /** - Called with normalized [request options](https://github.com/sindresorhus/got#options). Got will make no further changes to the request before it is sent (except the body serialization). This is especially useful in conjunction with [`got.extend()`](https://github.com/sindresorhus/got#instances) when you want to create an API client that, for example, uses HMAC-signing. - - @see [AWS section](https://github.com/sindresorhus/got#aws) for an example. - @default [] - */ - beforeRequest?: BeforeRequestHook[]; - - /** - Called with normalized [request options](https://github.com/sindresorhus/got#options). Got will make no further changes to the request. This is especially useful when you want to avoid dead sites. - - @default [] - */ - beforeRedirect?: BeforeRedirectHook[]; - - /** - Called with normalized [request options](https://github.com/sindresorhus/got#options), the error and the retry count. Got will make no further changes to the request. This is especially useful when some extra work is required before the next try. - - @default [] - */ - beforeRetry?: BeforeRetryHook[]; - - /** - Called with an `Error` instance. The error is passed to the hook right before it's thrown. This is especially useful when you want to have more detailed errors. - - **Note:** Errors thrown while normalizing input options are thrown directly and not part of this hook. - - @default [] - */ - beforeError?: BeforeErrorHook[]; - - /** - Called with [response object](https://github.com/sindresorhus/got#response) and a retry function. - - Each function should return the response. This is especially useful when you want to refresh an access token. - - @default [] - */ - afterResponse?: AfterResponseHook[]; -} - -export type HookEvent = keyof Hooks; - -const knownHookEvents: readonly HookEvent[] = [ - 'beforeError', - 'init', - 'beforeRequest', - 'beforeRedirect', - 'beforeRetry', - 'afterResponse' -]; - -export default knownHookEvents; diff --git a/source/normalize-arguments.ts b/source/normalize-arguments.ts deleted file mode 100644 index 831b7981c..000000000 --- a/source/normalize-arguments.ts +++ /dev/null @@ -1,536 +0,0 @@ -import {URL, URLSearchParams} from 'url'; -import {promisify, deprecate} from 'util'; -import CacheableRequest = require('cacheable-request'); -import http = require('http'); -import https = require('https'); -import Keyv = require('keyv'); -import lowercaseKeys = require('lowercase-keys'); -import stream = require('stream'); -import toReadableStream = require('to-readable-stream'); -import is from '@sindresorhus/is'; -import CacheableLookup from 'cacheable-lookup'; -import {Merge} from 'type-fest'; -import {UnsupportedProtocolError} from './errors'; -import knownHookEvents, {InitHook} from './known-hook-events'; -import dynamicRequire from './utils/dynamic-require'; -import getBodySize from './utils/get-body-size'; -import isFormData from './utils/is-form-data'; -import merge from './utils/merge'; -import optionsToUrl from './utils/options-to-url'; -import supportsBrotli from './utils/supports-brotli'; -import { - AgentByProtocol, - Defaults, - Method, - NormalizedOptions, - Options, - RequestFunction, - URLOrOptions, - requestSymbol -} from './types'; - -// `preNormalizeArguments` normalizes these options: `headers`, `prefixUrl`, `hooks`, `timeout`, `retry` and `method`. -// `normalizeArguments` is *only* called on `got(...)`. It normalizes the URL and performs `mergeOptions(...)`. -// `normalizeRequestArguments` converts Got options into HTTP options. - -type NonEnumerableProperty = 'context' | 'body' | 'json' | 'form'; -const nonEnumerableProperties: NonEnumerableProperty[] = [ - 'context', - 'body', - 'json', - 'form' -]; - -const isAgentByProtocol = (agent: Options['agent']): agent is AgentByProtocol => is.object(agent); - -// TODO: `preNormalizeArguments` should merge `options` & `defaults` -export const preNormalizeArguments = (options: Options, defaults?: NormalizedOptions): NormalizedOptions => { - // `options.headers` - if (is.undefined(options.headers)) { - options.headers = {}; - } else { - options.headers = lowercaseKeys(options.headers); - } - - for (const [key, value] of Object.entries(options.headers)) { - if (is.null_(value)) { - throw new TypeError(`Use \`undefined\` instead of \`null\` to delete the \`${key}\` header`); - } - } - - // `options.prefixUrl` - if (is.urlInstance(options.prefixUrl) || is.string(options.prefixUrl)) { - options.prefixUrl = options.prefixUrl.toString(); - - if (options.prefixUrl.length !== 0 && !options.prefixUrl.endsWith('/')) { - options.prefixUrl += '/'; - } - } else { - options.prefixUrl = defaults ? defaults.prefixUrl : ''; - } - - // `options.hooks` - if (is.undefined(options.hooks)) { - options.hooks = {}; - } - - if (is.object(options.hooks)) { - for (const event of knownHookEvents) { - if (Reflect.has(options.hooks, event)) { - if (!is.array(options.hooks[event])) { - throw new TypeError(`Parameter \`${event}\` must be an Array, not ${is(options.hooks[event])}`); - } - } else { - options.hooks[event] = []; - } - } - } else { - throw new TypeError(`Parameter \`hooks\` must be an Object, not ${is(options.hooks)}`); - } - - if (defaults) { - for (const event of knownHookEvents) { - if (!(Reflect.has(options.hooks, event) && is.undefined(options.hooks[event]))) { - // @ts-ignore Union type array is not assignable to union array type - options.hooks[event] = [ - ...defaults.hooks[event], - ...options.hooks[event]! - ]; - } - } - } - - // `options.timeout` - if (is.number(options.timeout)) { - options.timeout = {request: options.timeout}; - } else if (!is.object(options.timeout)) { - options.timeout = {}; - } - - // `options.retry` - const {retry} = options; - - if (defaults) { - options.retry = {...defaults.retry}; - } else { - options.retry = { - calculateDelay: retryObject => retryObject.computedValue, - limit: 0, - methods: [], - statusCodes: [], - errorCodes: [], - maxRetryAfter: undefined - }; - } - - if (is.object(retry)) { - options.retry = { - ...options.retry, - ...retry - }; - } else if (is.number(retry)) { - options.retry.limit = retry; - } - - if (options.retry.maxRetryAfter === undefined) { - options.retry.maxRetryAfter = Math.min( - ...[options.timeout.request, options.timeout.connect].filter((n): n is number => !is.nullOrUndefined(n)) - ); - } - - options.retry.methods = [...new Set(options.retry.methods!.map(method => method.toUpperCase() as Method))]; - options.retry.statusCodes = [...new Set(options.retry.statusCodes)]; - options.retry.errorCodes = [...new Set(options.retry.errorCodes)]; - - // `options.dnsCache` - if (options.dnsCache && !(options.dnsCache instanceof CacheableLookup)) { - options.dnsCache = new CacheableLookup({cacheAdapter: options.dnsCache as Keyv}); - } - - // `options.method` - if (is.string(options.method)) { - options.method = options.method.toUpperCase() as Method; - } else { - options.method = defaults?.method ?? 'GET'; - } - - // Better memory management, so we don't have to generate a new object every time - if (options.cache) { - (options as NormalizedOptions).cacheableRequest = new CacheableRequest( - // @ts-ignore Cannot properly type a function with multiple definitions yet - (requestOptions, handler) => requestOptions[requestSymbol](requestOptions, handler), - options.cache - ); - } - - // `options.cookieJar` - if (is.object(options.cookieJar)) { - let {setCookie, getCookieString} = options.cookieJar; - - // Horrible `tough-cookie` check - if (setCookie.length === 4 && getCookieString.length === 0) { - if (!Reflect.has(setCookie, promisify.custom)) { - // @ts-ignore TS is dumb - it says `setCookie` is `never`. - setCookie = promisify(setCookie.bind(options.cookieJar)); - getCookieString = promisify(getCookieString.bind(options.cookieJar)); - } - } else if (setCookie.length !== 2) { - throw new TypeError('`options.cookieJar.setCookie` needs to be an async function with 2 arguments'); - } else if (getCookieString.length !== 1) { - throw new TypeError('`options.cookieJar.getCookieString` needs to be an async function with 1 argument'); - } - - options.cookieJar = {setCookie, getCookieString}; - } - - // `options.encoding` - if (is.null_(options.encoding)) { - throw new TypeError('To get a Buffer, set `options.responseType` to `buffer` instead'); - } - - // `options.maxRedirects` - if (!Reflect.has(options, 'maxRedirects') && !(defaults && Reflect.has(defaults, 'maxRedirects'))) { - options.maxRedirects = 0; - } - - // Merge defaults - if (defaults) { - options = merge({}, defaults, options); - } - - // `options._pagination` - if (is.object(options._pagination)) { - const {_pagination: pagination} = options; - - if (!is.function_(pagination.transform)) { - throw new TypeError('`options._pagination.transform` must be implemented'); - } - - if (!is.function_(pagination.shouldContinue)) { - throw new TypeError('`options._pagination.shouldContinue` must be implemented'); - } - - if (!is.function_(pagination.filter)) { - throw new TypeError('`options._pagination.filter` must be implemented'); - } - - if (!is.function_(pagination.paginate)) { - throw new TypeError('`options._pagination.paginate` must be implemented'); - } - } - - // Other values - options.decompress = Boolean(options.decompress); - options.isStream = Boolean(options.isStream); - options.throwHttpErrors = Boolean(options.throwHttpErrors); - options.ignoreInvalidCookies = Boolean(options.ignoreInvalidCookies); - options.cache = options.cache ?? false; - options.responseType = options.responseType ?? 'text'; - options.resolveBodyOnly = Boolean(options.resolveBodyOnly); - options.followRedirect = Boolean(options.followRedirect); - options.dnsCache = options.dnsCache ?? false; - options.useElectronNet = Boolean(options.useElectronNet); - options.methodRewriting = Boolean(options.methodRewriting); - options.allowGetBody = Boolean(options.allowGetBody); - options.context = options.context ?? {}; - - return options as NormalizedOptions; -}; - -export const mergeOptions = (...sources: Options[]): NormalizedOptions => { - let mergedOptions = preNormalizeArguments({}); - - // Non enumerable properties shall not be merged - const properties: Partial<{[Key in NonEnumerableProperty]: any}> = {}; - - for (const source of sources) { - mergedOptions = preNormalizeArguments(merge({}, source), mergedOptions); - - for (const name of nonEnumerableProperties) { - if (!Reflect.has(source, name)) { - continue; - } - - properties[name] = { - writable: true, - configurable: true, - enumerable: false, - value: source[name] - }; - } - } - - Object.defineProperties(mergedOptions, properties); - - return mergedOptions; -}; - -export const normalizeArguments = (url: URLOrOptions, options?: Options, defaults?: Defaults): NormalizedOptions => { - // Merge options - if (typeof url === 'undefined') { - throw new TypeError('Missing `url` argument'); - } - - const runInitHooks = (hooks?: InitHook[], options?: Options): void => { - if (hooks && options) { - for (const hook of hooks) { - const result = hook(options); - - if (is.promise(result)) { - throw new TypeError('The `init` hook must be a synchronous function'); - } - } - } - }; - - const hasUrl = is.urlInstance(url) || is.string(url); - if (hasUrl) { - if (options) { - if (Reflect.has(options, 'url')) { - throw new TypeError('The `url` option cannot be used if the input is a valid URL.'); - } - } else { - options = {}; - } - - // @ts-ignore URL is not URL - options.url = url; - - runInitHooks(defaults?.options.hooks.init, options); - runInitHooks(options.hooks?.init, options); - } else if (Reflect.has(url as object, 'resolve')) { - throw new Error('The legacy `url.Url` is deprecated. Use `URL` instead.'); - } else { - runInitHooks(defaults?.options.hooks.init, url as Options); - runInitHooks((url as Options).hooks?.init, url as Options); - - if (options) { - runInitHooks(defaults?.options.hooks.init, options); - runInitHooks(options.hooks?.init, options); - } - } - - if (hasUrl) { - options = mergeOptions(defaults?.options ?? {}, options ?? {}); - } else { - options = mergeOptions(defaults?.options ?? {}, url as object, options ?? {}); - } - - // Normalize URL - // TODO: drop `optionsToUrl` in Got 12 - if (is.string(options.url)) { - options.url = (options.prefixUrl as string) + options.url; - options.url = options.url.replace(/^unix:/, 'http://$&'); - - if (options.searchParams || options.search) { - options.url = options.url.split('?')[0]; - } - - // @ts-ignore URL is not URL - options.url = optionsToUrl({ - origin: options.url, - ...options - }); - } else if (!is.urlInstance(options.url)) { - // @ts-ignore URL is not URL - options.url = optionsToUrl({origin: options.prefixUrl as string, ...options}); - } - - const normalizedOptions = options as NormalizedOptions; - - // Make it possible to change `options.prefixUrl` - let prefixUrl = options.prefixUrl as string; - Object.defineProperty(normalizedOptions, 'prefixUrl', { - set: (value: string) => { - if (!normalizedOptions.url.href.startsWith(value)) { - throw new Error(`Cannot change \`prefixUrl\` from ${prefixUrl} to ${value}: ${normalizedOptions.url.href}`); - } - - normalizedOptions.url = new URL(value + normalizedOptions.url.href.slice(prefixUrl.length)); - prefixUrl = value; - }, - get: () => prefixUrl - }); - - // Make it possible to remove default headers - for (const [key, value] of Object.entries(normalizedOptions.headers)) { - if (is.undefined(value)) { - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete normalizedOptions.headers[key]; - } - } - - return normalizedOptions; -}; - -const withoutBody: ReadonlySet = new Set(['HEAD']); -const withoutBodyUnlessSpecified = 'GET'; - -export type NormalizedRequestArguments = Merge; -}>; - -export const normalizeRequestArguments = async (options: NormalizedOptions): Promise => { - options = mergeOptions(options); - - // Serialize body - const {headers} = options; - const hasNoContentType = is.undefined(headers['content-type']); - - { - // TODO: these checks should be moved to `preNormalizeArguments` - const isForm = !is.undefined(options.form); - const isJson = !is.undefined(options.json); - const isBody = !is.undefined(options.body); - if ((isBody || isForm || isJson) && withoutBody.has(options.method)) { - throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); - } - - if (!options.allowGetBody && (isBody || isForm || isJson) && withoutBodyUnlessSpecified === options.method) { - throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); - } - - if ([isBody, isForm, isJson].filter(isTrue => isTrue).length > 1) { - throw new TypeError('The `body`, `json` and `form` options are mutually exclusive'); - } - - if ( - isBody && - !is.nodeStream(options.body) && - !is.string(options.body) && - !is.buffer(options.body) && - !(is.object(options.body) && isFormData(options.body)) - ) { - throw new TypeError('The `body` option must be a stream.Readable, string or Buffer'); - } - - if (isForm && !is.object(options.form)) { - throw new TypeError('The `form` option must be an Object'); - } - } - - if (options.body) { - // Special case for https://github.com/form-data/form-data - if (is.object(options.body) && isFormData(options.body) && hasNoContentType) { - headers['content-type'] = `multipart/form-data; boundary=${options.body.getBoundary()}`; - } - } else if (options.form) { - if (hasNoContentType) { - headers['content-type'] = 'application/x-www-form-urlencoded'; - } - - options.body = (new URLSearchParams(options.form as Record)).toString(); - } else if (options.json) { - if (hasNoContentType) { - headers['content-type'] = 'application/json'; - } - - options.body = JSON.stringify(options.json); - } - - const uploadBodySize = await getBodySize(options); - - if (!is.nodeStream(options.body)) { - options.body = toReadableStream(options.body!); - } - - // See https://tools.ietf.org/html/rfc7230#section-3.3.2 - // A user agent SHOULD send a Content-Length in a request message when - // no Transfer-Encoding is sent and the request method defines a meaning - // for an enclosed payload body. For example, a Content-Length header - // field is normally sent in a POST request even when the value is 0 - // (indicating an empty payload body). A user agent SHOULD NOT send a - // Content-Length header field when the request message does not contain - // a payload body and the method semantics do not anticipate such a - // body. - if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding'])) { - if ( - (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH' || options.method === 'DELETE' || (options.allowGetBody && options.method === 'GET')) && - !is.undefined(uploadBodySize) - ) { - // @ts-ignore We assign if it is undefined, so this IS correct - headers['content-length'] = String(uploadBodySize); - } - } - - if (!options.isStream && options.responseType === 'json' && is.undefined(headers.accept)) { - headers.accept = 'application/json'; - } - - if (options.decompress && is.undefined(headers['accept-encoding'])) { - headers['accept-encoding'] = supportsBrotli ? 'gzip, deflate, br' : 'gzip, deflate'; - } - - // Validate URL - if (options.url.protocol !== 'http:' && options.url.protocol !== 'https:') { - throw new UnsupportedProtocolError(options); - } - - decodeURI(options.url.toString()); - - // Normalize request function - if (is.function_(options.request)) { - options[requestSymbol] = options.request; - delete options.request; - } else { - options[requestSymbol] = options.url.protocol === 'https:' ? https.request : http.request; - } - - // UNIX sockets - if (options.url.hostname === 'unix') { - const matches = /(?.+?):(?.+)/.exec(options.url.pathname); - - if (matches?.groups) { - const {socketPath, path} = matches.groups; - - options = { - ...options, - socketPath, - path, - host: '' - }; - } - } - - if (isAgentByProtocol(options.agent)) { - options.agent = options.agent[options.url.protocol.slice(0, -1) as keyof AgentByProtocol] ?? options.agent; - } - - if (options.dnsCache) { - options.lookup = options.dnsCache.lookup; - } - - /* istanbul ignore next: electron.net is broken */ - // No point in typing process.versions correctly, as - // `process.version.electron` is used only once, right here. - if (options.useElectronNet && (process.versions as any).electron) { - const electron = dynamicRequire(module, 'electron') as any; // Trick webpack - options.request = deprecate( - electron.net.request ?? electron.remote.net.request, - 'Electron support has been deprecated and will be removed in Got 11.\n' + - 'See https://github.com/sindresorhus/got/issues/899 for further information.', - 'GOT_ELECTRON' - ); - } - - // Got's `timeout` is an object, http's `timeout` is a number, so they're not compatible. - delete options.timeout; - - // Set cookies - if (options.cookieJar) { - const cookieString = await options.cookieJar.getCookieString(options.url.toString()); - - if (is.nonEmptyString(cookieString)) { - options.headers.cookie = cookieString; - } else { - delete options.headers.cookie; - } - } - - // `http-cache-semantics` checks this - delete options.url; - - return options as unknown as NormalizedRequestArguments; -}; diff --git a/source/progress.ts b/source/progress.ts deleted file mode 100644 index ec04e24e7..000000000 --- a/source/progress.ts +++ /dev/null @@ -1,48 +0,0 @@ -import EventEmitter = require('events'); -import {Transform as TransformStream} from 'stream'; -import is from '@sindresorhus/is'; - -export function createProgressStream(name: 'downloadProgress' | 'uploadProgress', emitter: EventEmitter, totalBytes?: number | string): TransformStream { - let transformedBytes = 0; - - if (is.string(totalBytes)) { - totalBytes = Number(totalBytes); - } - - const progressStream = new TransformStream({ - transform(chunk, _encoding, callback) { - transformedBytes += chunk.length; - - const percent = totalBytes ? transformedBytes / (totalBytes as number) : 0; - - // Let `flush()` be responsible for emitting the last event - if (percent < 1) { - emitter.emit(name, { - percent, - transferred: transformedBytes, - total: totalBytes - }); - } - - callback(undefined, chunk); - }, - - flush(callback) { - emitter.emit(name, { - percent: 1, - transferred: transformedBytes, - total: totalBytes - }); - - callback(); - } - }); - - emitter.emit(name, { - percent: 0, - transferred: 0, - total: totalBytes - }); - - return progressStream; -} diff --git a/source/request-as-event-emitter.ts b/source/request-as-event-emitter.ts deleted file mode 100644 index d10725bf9..000000000 --- a/source/request-as-event-emitter.ts +++ /dev/null @@ -1,345 +0,0 @@ -import {ReadStream} from 'fs'; -import CacheableRequest = require('cacheable-request'); -import EventEmitter = require('events'); -import http = require('http'); -import stream = require('stream'); -import {URL} from 'url'; -import {promisify} from 'util'; -import is from '@sindresorhus/is'; -import timer, {ClientRequestWithTimings} from '@szmarczak/http-timer'; -import {ProxyStream} from './as-stream'; -import calculateRetryDelay from './calculate-retry-delay'; -import {CacheError, GotError, MaxRedirectsError, RequestError, TimeoutError} from './errors'; -import getResponse from './get-response'; -import {normalizeRequestArguments} from './normalize-arguments'; -import {createProgressStream} from './progress'; -import timedOut, {TimeoutError as TimedOutTimeoutError} from './utils/timed-out'; -import {GeneralError, NormalizedOptions, Response, requestSymbol} from './types'; -import urlToOptions from './utils/url-to-options'; -import pEvent = require('p-event'); - -const setImmediateAsync = async (): Promise => new Promise(resolve => setImmediate(resolve)); -const pipeline = promisify(stream.pipeline); - -const redirectCodes: ReadonlySet = new Set([300, 301, 302, 303, 304, 307, 308]); - -export interface RequestAsEventEmitter extends EventEmitter { - retry: (error: TimeoutError | RequestError) => boolean; - abort: () => void; -} - -export default (options: NormalizedOptions): RequestAsEventEmitter => { - const emitter = new EventEmitter() as RequestAsEventEmitter; - - const requestUrl = options.url.toString(); - const redirects: string[] = []; - let retryCount = 0; - - let currentRequest: http.ClientRequest; - - // `request.aborted` is a boolean since v11.0.0: https://github.com/nodejs/node/commit/4b00c4fafaa2ae8c41c1f78823c0feb810ae4723#diff-e3bc37430eb078ccbafe3aa3b570c91a - const isAborted = (): boolean => typeof currentRequest.aborted === 'number' || (currentRequest.aborted as unknown as boolean); - - const emitError = async (error: GeneralError): Promise => { - try { - for (const hook of options.hooks.beforeError) { - // eslint-disable-next-line no-await-in-loop - error = await hook(error); - } - - emitter.emit('error', error); - } catch (error_) { - emitter.emit('error', error_); - } - }; - - const get = async (): Promise => { - let httpOptions = await normalizeRequestArguments(options); - - const handleResponse = async (response: http.IncomingMessage): Promise => { - try { - /* istanbul ignore next: fixes https://github.com/electron/electron/blob/cbb460d47628a7a146adf4419ed48550a98b2923/lib/browser/api/net.js#L59-L65 */ - if (options.useElectronNet) { - response = new Proxy(response, { - get: (target, name) => { - if (name === 'trailers' || name === 'rawTrailers') { - return []; - } - - const value = (target as any)[name]; - return is.function_(value) ? value.bind(target) : value; - } - }); - } - - const typedResponse = response as Response; - const {statusCode} = typedResponse; - typedResponse.statusMessage = is.nonEmptyString(typedResponse.statusMessage) ? typedResponse.statusMessage : http.STATUS_CODES[statusCode]; - typedResponse.url = options.url.toString(); - typedResponse.requestUrl = requestUrl; - typedResponse.retryCount = retryCount; - typedResponse.redirectUrls = redirects; - typedResponse.request = {options}; - typedResponse.isFromCache = typedResponse.fromCache ?? false; - delete typedResponse.fromCache; - - if (!typedResponse.isFromCache) { - typedResponse.ip = response.socket.remoteAddress!; - } - - const rawCookies = typedResponse.headers['set-cookie']; - if (Reflect.has(options, 'cookieJar') && rawCookies) { - let promises: Array> = rawCookies.map(async (rawCookie: string) => options.cookieJar.setCookie(rawCookie, typedResponse.url)); - - if (options.ignoreInvalidCookies) { - promises = promises.map(async p => p.catch(() => {})); - } - - await Promise.all(promises); - } - - if (options.followRedirect && Reflect.has(typedResponse.headers, 'location') && redirectCodes.has(statusCode)) { - typedResponse.resume(); // We're being redirected, we don't care about the response. - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-boolean-literal-compare - if (statusCode === 303 || options.methodRewriting === false) { - if (options.method !== 'GET' && options.method !== 'HEAD') { - // Server responded with "see other", indicating that the resource exists at another location, - // and the client should request it from that location via GET or HEAD. - options.method = 'GET'; - } - - if (Reflect.has(options, 'body')) { - delete options.body; - } - - if (Reflect.has(options, 'json')) { - delete options.json; - } - - if (Reflect.has(options, 'form')) { - delete options.form; - } - } - - if (redirects.length >= options.maxRedirects) { - throw new MaxRedirectsError(typedResponse, options.maxRedirects, options); - } - - // Handles invalid URLs. See https://github.com/sindresorhus/got/issues/604 - const redirectBuffer = Buffer.from(typedResponse.headers.location, 'binary').toString(); - const redirectUrl = new URL(redirectBuffer, options.url); - - // Redirecting to a different site, clear cookies. - if (redirectUrl.hostname !== options.url.hostname && Reflect.has(options.headers, 'cookie')) { - delete options.headers.cookie; - } - - redirects.push(redirectUrl.toString()); - options.url = redirectUrl; - - for (const hook of options.hooks.beforeRedirect) { - // eslint-disable-next-line no-await-in-loop - await hook(options, typedResponse); - } - - emitter.emit('redirect', response, options); - - await get(); - return; - } - - await getResponse(typedResponse, options, emitter); - } catch (error) { - emitError(error); - } - }; - - const handleRequest = async (request: ClientRequestWithTimings): Promise => { - let isPiped = false; - let isFinished = false; - - // `request.finished` doesn't indicate whether this has been emitted or not - request.once('finish', () => { - isFinished = true; - }); - - currentRequest = request; - - const onError = (error: GeneralError): void => { - if (error instanceof TimedOutTimeoutError) { - error = new TimeoutError(error, request.timings!, options); - } else { - error = new RequestError(error, options); - } - - if (!emitter.retry(error as GotError)) { - emitError(error); - } - }; - - request.on('error', error => { - if (isPiped) { - // Check if it's caught by `stream.pipeline(...)` - if (!isFinished) { - return; - } - - // We need to let `TimedOutTimeoutError` through, because `stream.pipeline(…)` aborts the request automatically. - if (isAborted() && !(error instanceof TimedOutTimeoutError)) { - return; - } - } - - onError(error); - }); - - try { - timer(request); - timedOut(request, options.timeout, options.url); - - emitter.emit('request', request); - - const uploadStream = createProgressStream('uploadProgress', emitter, httpOptions.headers!['content-length'] as string); - - isPiped = true; - - await pipeline( - httpOptions.body!, - uploadStream, - request - ); - - request.emit('upload-complete'); - } catch (error) { - if (isAborted() && error.message === 'Premature close') { - // The request was aborted on purpose - return; - } - - onError(error); - } - }; - - if (options.cache) { - // `cacheable-request` doesn't support Node 10 API, fallback. - httpOptions = { - ...httpOptions, - ...urlToOptions(options.url) - }; - - // @ts-ignore `cacheable-request` has got invalid types - const cacheRequest = options.cacheableRequest!(httpOptions, handleResponse); - - cacheRequest.once('error', (error: GeneralError) => { - if (error instanceof CacheableRequest.RequestError) { - emitError(new RequestError(error, options)); - } else { - emitError(new CacheError(error, options)); - } - }); - - cacheRequest.once('request', handleRequest); - } else { - // Catches errors thrown by calling `requestFn(…)` - try { - handleRequest(httpOptions[requestSymbol](options.url, httpOptions, handleResponse)); - } catch (error) { - emitError(new RequestError(error, options)); - } - } - }; - - emitter.retry = error => { - let backoff: number; - - retryCount++; - - try { - backoff = options.retry.calculateDelay({ - attemptCount: retryCount, - retryOptions: options.retry, - error, - computedValue: calculateRetryDelay({ - attemptCount: retryCount, - retryOptions: options.retry, - error, - computedValue: 0 - }) - }); - } catch (error_) { - emitError(error_); - return false; - } - - if (backoff) { - const retry = async (options: NormalizedOptions): Promise => { - try { - for (const hook of options.hooks.beforeRetry) { - // eslint-disable-next-line no-await-in-loop - await hook(options, error, retryCount); - } - - await get(); - } catch (error_) { - emitError(error_); - } - }; - - setTimeout(retry, backoff, {...options, forceRefresh: true}); - return true; - } - - return false; - }; - - emitter.abort = () => { - emitter.prependListener('request', (request: http.ClientRequest) => { - request.abort(); - }); - - if (currentRequest) { - currentRequest.abort(); - } - }; - - (async () => { - try { - if (options.body instanceof ReadStream) { - await pEvent(options.body, 'open'); - } - - // Promises are executed immediately. - // If there were no `setImmediate` here, - // `promise.json()` would have no effect - // as the request would be sent already. - await setImmediateAsync(); - - for (const hook of options.hooks.beforeRequest) { - // eslint-disable-next-line no-await-in-loop - await hook(options); - } - - await get(); - } catch (error) { - emitError(error); - } - })(); - - return emitter; -}; - -export const proxyEvents = (proxy: EventEmitter | ProxyStream, emitter: RequestAsEventEmitter): void => { - const events = [ - 'request', - 'redirect', - 'uploadProgress', - 'downloadProgress' - ]; - - for (const event of events) { - emitter.on(event, (...args: unknown[]) => { - proxy.emit(event, ...args); - }); - } -}; diff --git a/source/types.ts b/source/types.ts index e9803f872..551def48a 100644 --- a/source/types.ts +++ b/source/types.ts @@ -1,274 +1,125 @@ -import http = require('http'); -import https = require('https'); -import Keyv = require('keyv'); -import CacheableRequest = require('cacheable-request'); -import PCancelable = require('p-cancelable'); -import ResponseLike = require('responselike'); import {URL} from 'url'; -import {Readable as ReadableStream} from 'stream'; -import {Timings, IncomingMessageWithTimings} from '@szmarczak/http-timer'; -import CacheableLookup from 'cacheable-lookup'; -import {Except, Merge} from 'type-fest'; -import {GotReturn} from './create'; -import {GotError, HTTPError, MaxRedirectsError, ParseError, TimeoutError, RequestError} from './errors'; -import {Hooks} from './known-hook-events'; -import {URLOptions} from './utils/options-to-url'; - -export type GeneralError = Error | GotError | HTTPError | MaxRedirectsError | ParseError; - -export type Method = - | 'GET' - | 'POST' - | 'PUT' - | 'PATCH' - | 'HEAD' - | 'DELETE' - | 'OPTIONS' - | 'TRACE' - | 'get' - | 'post' - | 'put' - | 'patch' - | 'head' - | 'delete' - | 'options' - | 'trace'; - -export type ResponseType = 'json' | 'buffer' | 'text'; - -export interface Response extends IncomingMessageWithTimings { - body: BodyType; - statusCode: number; - - /** - The remote IP address. - - Note: Not available when the response is cached. This is hopefully a temporary limitation, see [lukechilds/cacheable-request#86](https://github.com/lukechilds/cacheable-request/issues/86). - */ - ip: string; - - fromCache?: boolean; - isFromCache?: boolean; - req?: http.ClientRequest; - requestUrl: string; - retryCount: number; - timings: Timings; - redirectUrls: string[]; - request: { - options: NormalizedOptions; - }; - url: string; -} - -// TODO: The `ResponseLike` type should be properly fixed instead: -// https://github.com/sindresorhus/got/pull/827/files#r323633794 -export interface ResponseObject extends Partial { - socket: { - remoteAddress: string; - }; -} - -export interface RetryObject { - attemptCount: number; - retryOptions: Required; - error: TimeoutError | RequestError; - computedValue: number; +import {CancelError} from 'p-cancelable'; +import { + // Request & Response + CancelableRequest, + Response, + + // Options + Options, + NormalizedOptions, + Defaults as DefaultOptions, + PaginationOptions, + + // Errors + ParseError, + RequestError, + CacheError, + ReadError, + HTTPError, + MaxRedirectsError, + TimeoutError +} from './as-promise'; +import Request from './core'; + +// `type-fest` utilities +type Except = Pick>; +type Merge = Except> & SecondType; + +export interface InstanceDefaults { + options: DefaultOptions; + handlers: HandlerFunction[]; + mutableDefaults: boolean; + _rawHandlers?: HandlerFunction[]; } -export type RetryFunction = (retryObject: RetryObject) => number; - +export type GotReturn = Request | CancelableRequest; export type HandlerFunction = (options: NormalizedOptions, next: (options: NormalizedOptions) => T) => T | Promise; -export interface DefaultRetryOptions { - limit: number; - methods: Method[]; - statusCodes: number[]; - errorCodes: string[]; - calculateDelay: RetryFunction; - maxRetryAfter?: number; -} - -export interface RetryOptions extends Partial { - retries?: number; -} - -export type RequestFunction = typeof http.request; - -export interface AgentByProtocol { - http?: http.Agent; - https?: https.Agent; -} - -export interface Delays { - lookup?: number; - connect?: number; - secureConnect?: number; - socket?: number; - response?: number; - send?: number; - request?: number; -} - -export type Headers = Record; - -interface ToughCookieJar { - getCookieString(currentUrl: string, options: {[key: string]: unknown}, cb: (err: Error | null, cookies: string) => void): void; - getCookieString(url: string, callback: (error: Error | null, cookieHeader: string) => void): void; - setCookie(cookieOrString: unknown, currentUrl: string, options: {[key: string]: unknown}, cb: (err: Error | null, cookie: unknown) => void): void; - setCookie(rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void): void; -} - -interface PromiseCookieJar { - getCookieString(url: string): Promise; - setCookie(rawCookie: string, url: string): Promise; +export interface ExtendOptions extends Options { + handlers?: HandlerFunction[]; + mutableDefaults?: boolean; } -export const requestSymbol = Symbol('request'); +export type OptionsOfTextResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType?: 'text'}; +export type OptionsOfJSONResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType: 'json'}; +export type OptionsOfBufferResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType: 'buffer'}; +export type StrictOptions = Except; +type ResponseBodyOnly = {resolveBodyOnly: true}; -/* eslint-disable @typescript-eslint/indent */ -export type DefaultOptions = Merge< - Required< - Except< - GotOptions, - // Override - 'hooks' | - 'retry' | - 'timeout' | - 'context' | - '_pagination' | +export type OptionsWithPagination = Merge>; - // Should not be present - 'agent' | - 'body' | - 'cookieJar' | - 'encoding' | - 'form' | - 'json' | - 'lookup' | - 'request' | - 'url' | - typeof requestSymbol - > - >, - { - hooks: Required; - retry: DefaultRetryOptions; - timeout: Delays; - context: {[key: string]: any}; - _pagination?: PaginationOptions['_pagination']; - } ->; -/* eslint-enable @typescript-eslint/indent */ +export interface GotPaginate { + (url: string | URL, options?: OptionsWithPagination): AsyncIterableIterator; + all(url: string | URL, options?: OptionsWithPagination): Promise; -export interface PaginationOptions { - _pagination?: { - transform?: (response: Response) => Promise | T[]; - filter?: (item: T, allItems: T[], currentItems: T[]) => boolean; - paginate?: (response: Response, allItems: T[], currentItems: T[]) => Options | false; - shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean; - countLimit?: number; - }; + // A bug. + // eslint-disable-next-line @typescript-eslint/adjacent-overload-signatures + (options?: OptionsWithPagination): AsyncIterableIterator; + // A bug. + // eslint-disable-next-line @typescript-eslint/adjacent-overload-signatures + all(options?: OptionsWithPagination): Promise; } -export interface GotOptions extends PaginationOptions { - [requestSymbol]?: RequestFunction; - url?: URL | string; - body?: string | Buffer | ReadableStream; - hooks?: Hooks; - decompress?: boolean; - isStream?: boolean; - encoding?: BufferEncoding; - method?: Method; - retry?: RetryOptions | number; - throwHttpErrors?: boolean; - cookieJar?: ToughCookieJar | PromiseCookieJar; - ignoreInvalidCookies?: boolean; - request?: RequestFunction; - agent?: http.Agent | https.Agent | boolean | AgentByProtocol; - cache?: string | CacheableRequest.StorageAdapter | false; - headers?: Headers; - responseType?: ResponseType; - resolveBodyOnly?: boolean; - followRedirect?: boolean; - prefixUrl?: URL | string; - timeout?: number | Delays; - dnsCache?: CacheableLookup | Map | Keyv | false; - useElectronNet?: boolean; - form?: {[key: string]: any}; - json?: {[key: string]: any}; - context?: {[key: string]: any}; - maxRedirects?: number; - lookup?: CacheableLookup['lookup']; - allowGetBody?: boolean; - methodRewriting?: boolean; -} +export interface GotRequestFunction { + // `asPromise` usage + (url: string | URL, options?: OptionsOfTextResponseBody): CancelableRequest>; + (url: string | URL, options?: OptionsOfJSONResponseBody): CancelableRequest>; + (url: string | URL, options?: OptionsOfBufferResponseBody): CancelableRequest>; -export type Options = Merge>; + (options: OptionsOfTextResponseBody): CancelableRequest>; + (options: OptionsOfJSONResponseBody): CancelableRequest>; + (options: OptionsOfBufferResponseBody): CancelableRequest>; -export interface NormalizedOptions extends Options { - // Normalized Got options - headers: Headers; - hooks: Required; - timeout: Delays; - dnsCache: CacheableLookup | false; - lookup?: CacheableLookup['lookup']; - retry: Required; - prefixUrl: string; - method: Method; - url: URL; - cacheableRequest?: (options: string | URL | http.RequestOptions, callback?: (response: http.ServerResponse | ResponseLike) => void) => CacheableRequest.Emitter; - cookieJar?: PromiseCookieJar; - maxRedirects: number; - pagination?: Required['_pagination']>; - [requestSymbol]: RequestFunction; + // `resolveBodyOnly` usage + (url: string | URL, options?: (OptionsOfTextResponseBody & ResponseBodyOnly)): CancelableRequest; + (url: string | URL, options?: (OptionsOfJSONResponseBody & ResponseBodyOnly)): CancelableRequest; + (url: string | URL, options?: (OptionsOfBufferResponseBody & ResponseBodyOnly)): CancelableRequest; - // Other values - decompress: boolean; - isStream: boolean; - throwHttpErrors: boolean; - ignoreInvalidCookies: boolean; - cache: CacheableRequest.StorageAdapter | false; - responseType: ResponseType; - resolveBodyOnly: boolean; - followRedirect: boolean; - useElectronNet: boolean; - methodRewriting: boolean; - allowGetBody: boolean; - context: {[key: string]: any}; + (options: (OptionsOfTextResponseBody & ResponseBodyOnly)): CancelableRequest; + (options: (OptionsOfJSONResponseBody & ResponseBodyOnly)): CancelableRequest; + (options: (OptionsOfBufferResponseBody & ResponseBodyOnly)): CancelableRequest; - // UNIX socket support - path?: string; -} + // `asStream` usage + (url: string | URL, options?: Options & {isStream: true}): Request; -export interface ExtendOptions extends Options { - handlers?: HandlerFunction[]; - mutableDefaults?: boolean; -} + (options: Options & {isStream: true}): Request; -export interface Defaults { - options: DefaultOptions; - handlers: HandlerFunction[]; - mutableDefaults: boolean; - _rawHandlers?: HandlerFunction[]; -} - -export type URLOrOptions = Options | string; + // Fallback + (url: string | URL, options?: Options): CancelableRequest | Request; -export interface Progress { - percent: number; - transferred: number; - total?: number; + (options: Options): CancelableRequest | Request; } -export interface GotEvents { - on(name: 'request', listener: (request: http.ClientRequest) => void): T; - on(name: 'response', listener: (response: Response) => void): T; - on(name: 'redirect', listener: (response: Response, nextOptions: NormalizedOptions) => void): T; - on(name: 'uploadProgress' | 'downloadProgress', listener: (progress: Progress) => void): T; -} - -export interface CancelableRequest extends PCancelable, GotEvents> { - json(): CancelableRequest; - buffer(): CancelableRequest; - text(): CancelableRequest; +export type HTTPAlias = + | 'get' + | 'post' + | 'put' + | 'patch' + | 'head' + | 'delete'; + +interface GotStreamFunction { + (url: string | URL, options?: Options & {isStream?: true}): Request; + (options?: Options & {isStream?: true}): Request; +} + +export type GotStream = GotStreamFunction & Record; + +export interface Got extends Record, GotRequestFunction { + stream: GotStream; + paginate: GotPaginate; + defaults: InstanceDefaults; + CacheError: typeof CacheError; + RequestError: typeof RequestError; + ReadError: typeof ReadError; + ParseError: typeof ParseError; + HTTPError: typeof HTTPError; + MaxRedirectsError: typeof MaxRedirectsError; + TimeoutError: typeof TimeoutError; + CancelError: typeof CancelError; + + extend(...instancesOrOptions: Array): Got; + mergeInstances(parent: Got, ...instances: Got[]): Got; + mergeOptions(...sources: Options[]): NormalizedOptions; } diff --git a/source/types/reflect/index.d.ts b/source/types/reflect/index.d.ts deleted file mode 100644 index c4625aabf..000000000 --- a/source/types/reflect/index.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -// The type-guarding behaviour is currently not supported as of TypeScript 3.7 -// https://github.com/microsoft/TypeScript/issues/30688 -declare namespace Reflect { - function has(target: T, propertyKey: Key): target is Required>; -} diff --git a/source/utils/dynamic-require.ts b/source/utils/dynamic-require.ts deleted file mode 100644 index 515f71995..000000000 --- a/source/utils/dynamic-require.ts +++ /dev/null @@ -1,3 +0,0 @@ -/* istanbul ignore file: used for webpack */ - -export default (moduleObject: NodeModule, moduleId: string): unknown => moduleObject.require(moduleId); diff --git a/source/utils/is-form-data.ts b/source/utils/is-form-data.ts deleted file mode 100644 index 3a299dddb..000000000 --- a/source/utils/is-form-data.ts +++ /dev/null @@ -1,4 +0,0 @@ -import FormData = require('form-data'); -import is from '@sindresorhus/is'; - -export default (body: unknown): body is FormData => is.nodeStream(body) && is.function_((body as FormData).getBoundary); diff --git a/source/utils/merge.ts b/source/utils/merge.ts deleted file mode 100644 index 9e6ccce67..000000000 --- a/source/utils/merge.ts +++ /dev/null @@ -1,32 +0,0 @@ -import {URL} from 'url'; -import is from '@sindresorhus/is'; -import {Merge} from 'type-fest'; - -export default function merge(target: Target, ...sources: Source[]): Merge { - for (const source of sources) { - for (const [key, sourceValue] of Object.entries(source)) { - const targetValue = target[key]; - - if (is.urlInstance(targetValue) && is.string(sourceValue)) { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = new URL(sourceValue, targetValue); - } else if (is.plainObject(sourceValue)) { - if (is.plainObject(targetValue)) { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = merge({}, targetValue, sourceValue); - } else { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = merge({}, sourceValue); - } - } else if (is.array(sourceValue)) { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = sourceValue.slice(); - } else { - // @ts-ignore TS doesn't recognise Target accepts string keys - target[key] = sourceValue; - } - } - } - - return target as Merge; -} diff --git a/source/utils/options-to-url.ts b/source/utils/options-to-url.ts deleted file mode 100644 index 777237b32..000000000 --- a/source/utils/options-to-url.ts +++ /dev/null @@ -1,113 +0,0 @@ -import {URL, URLSearchParams} from 'url'; - -function validateSearchParams(searchParams: Record): asserts searchParams is Record { - for (const value of Object.values(searchParams)) { - if (typeof value !== 'string' && typeof value !== 'number' && typeof value !== 'boolean' && value !== null) { - throw new TypeError(`The \`searchParams\` value '${String(value)}' must be a string, number, boolean or null`); - } - } -} - -export interface URLOptions { - href?: string; - origin?: string; - protocol?: string; - username?: string; - password?: string; - host?: string; - hostname?: string; - port?: string | number; - pathname?: string; - search?: string; - searchParams?: Record | URLSearchParams | string; - hash?: string; - - // The only accepted legacy URL options - path?: string; -} - -const keys: Array> = [ - 'protocol', - 'username', - 'password', - 'host', - 'hostname', - 'port', - 'pathname', - 'search', - 'hash' -]; - -export default (options: URLOptions): URL => { - let origin: string; - - if (options.path) { - if (options.pathname) { - throw new TypeError('Parameters `path` and `pathname` are mutually exclusive.'); - } - - if (options.search) { - throw new TypeError('Parameters `path` and `search` are mutually exclusive.'); - } - - if (options.searchParams) { - throw new TypeError('Parameters `path` and `searchParams` are mutually exclusive.'); - } - } - - if (Reflect.has(options, 'auth')) { - throw new TypeError('Parameter `auth` is deprecated. Use `username` / `password` instead.'); - } - - if (options.search && options.searchParams) { - throw new TypeError('Parameters `search` and `searchParams` are mutually exclusive.'); - } - - if (options.href) { - return new URL(options.href); - } - - if (options.origin) { - origin = options.origin; - } else { - if (!options.protocol) { - throw new TypeError('No URL protocol specified'); - } - - origin = `${options.protocol}//${options.hostname ?? options.host ?? ''}`; - } - - const url = new URL(origin); - - if (options.path) { - const searchIndex = options.path.indexOf('?'); - if (searchIndex === -1) { - options.pathname = options.path; - } else { - options.pathname = options.path.slice(0, searchIndex); - options.search = options.path.slice(searchIndex + 1); - } - } - - if (Reflect.has(options, 'path')) { - delete options.path; - } - - for (const key of keys) { - if (Reflect.has(options, key)) { - url[key] = options[key].toString(); - } - } - - if (options.searchParams) { - if (typeof options.searchParams !== 'string' && !(options.searchParams instanceof URLSearchParams)) { - validateSearchParams(options.searchParams); - } - - (new URLSearchParams(options.searchParams as Record)).forEach((value, key) => { - url.searchParams.append(key, value); - }); - } - - return url; -}; diff --git a/source/utils/supports-brotli.ts b/source/utils/supports-brotli.ts deleted file mode 100644 index 58b4ee283..000000000 --- a/source/utils/supports-brotli.ts +++ /dev/null @@ -1,3 +0,0 @@ -import zlib = require('zlib'); - -export default typeof zlib.createBrotliDecompress === 'function'; diff --git a/test/agent.ts b/test/agent.ts index 149588c34..61aab68cc 100644 --- a/test/agent.ts +++ b/test/agent.ts @@ -47,7 +47,9 @@ test('non-object agent option works with http', withServer, async (t, server, go t.truthy((await got({ rejectUnauthorized: false, - agent + agent: { + http: agent + } })).body); t.true(spy.calledOnce); @@ -64,7 +66,9 @@ test('non-object agent option works with https', withServer, async (t, server, g t.truthy((await got.secure({ rejectUnauthorized: false, - agent + agent: { + https: agent + } })).body); t.true(spy.calledOnce); @@ -126,7 +130,9 @@ test('socket connect listener cleaned up after request', withServer, async (t, s // eslint-disable-next-line no-await-in-loop await got.secure({ rejectUnauthorized: false, - agent + agent: { + https: agent + } }); } diff --git a/test/arguments.ts b/test/arguments.ts index 9ed09fd29..e519c695a 100644 --- a/test/arguments.ts +++ b/test/arguments.ts @@ -3,7 +3,7 @@ import {parse, URL, URLSearchParams} from 'url'; import test from 'ava'; import {Handler} from 'express'; import pEvent = require('p-event'); -import got from '../source'; +import got, {StrictOptions} from '../source'; import withServer from './helpers/with-server'; const echoUrl: Handler = (request, response) => { @@ -12,10 +12,17 @@ const echoUrl: Handler = (request, response) => { test('`url` is required', async t => { await t.throwsAsync( - // @ts-ignore Error tests got(''), { - instanceOf: TypeError, + message: 'Missing `url` property' + } + ); + + await t.throwsAsync( + got({ + url: '' + }), + { message: 'No URL protocol specified' } ); @@ -33,8 +40,7 @@ test('`url` should be utf-8 encoded', async t => { test('throws if no arguments provided', async t => { // @ts-ignore Error tests await t.throwsAsync(got(), { - instanceOf: TypeError, - message: 'Missing `url` argument' + message: 'Missing `url` property' }); }); @@ -45,13 +51,7 @@ test('throws an error if the protocol is not specified', async t => { }); await t.throwsAsync(got({}), { - instanceOf: TypeError, - message: 'No URL protocol specified' - }); - - await t.throwsAsync(got({}), { - instanceOf: TypeError, - message: 'No URL protocol specified' + message: 'Missing `url` property' }); }); @@ -60,7 +60,7 @@ test('string url with searchParams is preserved', withServer, async (t, server, const path = '?test=http://example.com?foo=bar'; const {body} = await got(path); - t.is(body, `/${path}`); + t.is(body, '/?test=http%3A%2F%2Fexample.com%3Ffoo%3Dbar'); }); test('options are optional', withServer, async (t, server, got) => { @@ -89,14 +89,38 @@ test('methods are normalized', withServer, async (t, server, got) => { await instance('test', {method: 'post'}); }); -test('throws an error when legacy URL is passed', withServer, async (t, server, got) => { +test.failing('throws an error when legacy URL is passed', withServer, async (t, server, got) => { server.get('/test', echoUrl); await t.throwsAsync( // @ts-ignore Error tests - got(parse(`${server.url}/test`)), - {message: 'The legacy `url.Url` is deprecated. Use `URL` instead.'} + got(parse(`${server.url}/test`), {prefixUrl: ''}), + {message: 'The legacy `url.Url` has been deprecated. Use `URL` instead.'} ); + + await t.throwsAsync( + got({ + protocol: 'http:', + hostname: 'localhost', + port: server.port, + prefixUrl: '' + }), + {message: 'The legacy `url.Url` has been deprecated. Use `URL` instead.'} + ); +}); + +test('accepts legacy URL options', withServer, async (t, server, got) => { + server.get('/test', echoUrl); + + const {body: secondBody} = await got({ + protocol: 'http:', + hostname: 'localhost', + port: server.port, + pathname: '/test', + prefixUrl: '' + }); + + t.is(secondBody, '/test'); }); test('overrides `searchParams` from options', withServer, async (t, server, got) => { @@ -107,20 +131,6 @@ test('overrides `searchParams` from options', withServer, async (t, server, got) { searchParams: { test: 'wow' - }, - cache: { - get(key: string) { - t.is(key, `cacheable-request:GET:${server.url}/?test=wow`); - }, - set(key: string) { - t.is(key, `cacheable-request:GET:${server.url}/?test=wow`); - }, - delete() { - return true; - }, - clear() { - return undefined; - } } } ); @@ -157,7 +167,6 @@ test('ignores empty searchParams object', withServer, async (t, server, got) => test('throws when passing body with a non payload method', async t => { // @ts-ignore Error tests await t.throwsAsync(got('https://example.com', {body: 'asdf'}), { - instanceOf: TypeError, message: 'The `GET` method cannot be used with a body' }); }); @@ -202,8 +211,7 @@ test('throws TypeError when `options.hooks` is not an object', async t => { // @ts-ignore Error tests got('https://example.com', {hooks: 'not object'}), { - instanceOf: TypeError, - message: 'Parameter `hooks` must be an Object, not string' + message: 'Expected value which is `predicate returns truthy for any value`, received value of type `Array`.' } ); }); @@ -213,8 +221,7 @@ test('throws TypeError when known `options.hooks` value is not an array', async // @ts-ignore Error tests got('https://example.com', {hooks: {beforeRequest: {}}}), { - instanceOf: TypeError, - message: 'Parameter `beforeRequest` must be an Array, not Object' + message: 'Parameter `beforeRequest` must be an Array, got Object' } ); }); @@ -225,7 +232,6 @@ test('throws TypeError when known `options.hooks` array item is not a function', // @ts-ignore Error tests got('https://example.com', {hooks: {beforeRequest: [{}]}}), { - instanceOf: TypeError, message: 'hook is not a function' } ); @@ -368,5 +374,111 @@ test('throws if `options.encoding` is `null`', async t => { test('`url` option and input argument are mutually exclusive', async t => { await t.throwsAsync(got('https://example.com', { url: 'https://example.com' - }), {message: 'The `url` option cannot be used if the input is a valid URL.'}); + }), {message: 'The `url` option is mutually exclusive with the `input` argument'}); +}); + +test('throws a helpful error when passing `followRedirects`', async t => { + await t.throwsAsync(got('https://example.com', { + // @ts-ignore For testing purposes + followRedirects: true + }), {message: 'The `followRedirects` option does not exist. Use `followRedirect` instead.'}); +}); + +test('merges `searchParams` instances', t => { + const instance = got.extend({ + searchParams: new URLSearchParams('a=1') + }, { + searchParams: new URLSearchParams('b=2') + }); + + t.is(instance.defaults.options.searchParams!.get('a'), '1'); + t.is(instance.defaults.options.searchParams!.get('b'), '2'); +}); + +test('throws a helpful error when passing `auth`', async t => { + await t.throwsAsync(got('https://example.com', { + // @ts-ignore For testing purposes + auth: 'username:password' + }), { + message: 'Parameter `auth` is deprecated. Use `username` / `password` instead.' + }); +}); + +test('throws on leading slashes', async t => { + await t.throwsAsync(got('/asdf', {prefixUrl: 'https://example.com'}), { + message: '`input` must not start with a slash when using `prefixUrl`' + }); +}); + +test('throws on invalid `dnsCache` option', async t => { + // @ts-ignore Error tests + await t.throwsAsync(got('https://example.com', { + dnsCache: 123 + }), {message: 'Parameter `dnsCache` must be a CacheableLookup instance or a boolean, got number'}); +}); + +test('throws on invalid `agent` option', async t => { + await t.throwsAsync(got('https://example.com', { + agent: { + // @ts-ignore Error tests + asdf: 123 + } + }), {message: 'Expected the `options.agent` properties to be `http`, `https` or `http2`, got `asdf`'}); +}); + +test('fallbacks to native http if `request(...)` returns undefined', withServer, async (t, server, got) => { + server.get('/', echoUrl); + + const {body} = await got('', {request: () => undefined}); + + t.is(body, '/'); +}); + +test('strict options', withServer, async (t, server, got) => { + server.get('/', echoUrl); + + const options: StrictOptions = {}; + + const {body} = await got(options); + + t.is(body, '/'); +}); + +test('does not throw on frozen options', withServer, async (t, server, got) => { + server.get('/', echoUrl); + + const options: StrictOptions = {}; + + Object.freeze(options); + + const {body} = await got(options); + + t.is(body, '/'); +}); + +test('normalizes search params included in input', t => { + const {url} = got.mergeOptions({ + url: new URL('https://example.com/?a=b c') + }); + + t.is(url.search, '?a=b+c'); +}); + +test('reuse options while using init hook', withServer, async (t, server, got) => { + t.plan(2); + + server.get('/', echoUrl); + + const options = { + hooks: { + init: [ + () => { + t.pass(); + } + ] + } + }; + + await got('', options); + await got('', options); }); diff --git a/test/cache.ts b/test/cache.ts index 4e49cc75f..d909ed8f2 100644 --- a/test/cache.ts +++ b/test/cache.ts @@ -4,6 +4,7 @@ import getStream = require('get-stream'); import {Handler} from 'express'; import {Response} from '../source'; import withServer from './helpers/with-server'; +import CacheableLookup from 'cacheable-lookup'; const cacheEndpoint: Handler = (_request, response) => { response.setHeader('Cache-Control', 'public, max-age=60'); @@ -81,11 +82,16 @@ test('redirects are cached and re-used internally', withServer, async (t, server server.get('/', cacheEndpoint); const cache = new Map(); - const firstResponse = await got('301', {cache}); - const secondResponse = await got('302', {cache}); + const A1 = await got('301', {cache}); + const B1 = await got('302', {cache}); + + const A2 = await got('301', {cache}); + const B2 = await got('302', {cache}); t.is(cache.size, 3); - t.is(firstResponse.body, secondResponse.body); + t.is(A1.body, B1.body); + t.is(A1.body, A2.body); + t.is(B1.body, B2.body); }); test('cached response has got options', withServer, async (t, server, got) => { @@ -135,10 +141,22 @@ test('doesn\'t cache response when received HTTP error', withServer, async (t, s }); test('DNS cache works', withServer, async (t, _server, got) => { - const map = new Map(); - await t.notThrowsAsync(got('https://example.com', {dnsCache: map, prefixUrl: ''})); + const instance = got.extend({ + dnsCache: true, + prefixUrl: '' + }); + + await t.notThrowsAsync(instance('https://example.com')); + + // @ts-ignore + t.is(instance.defaults.options.dnsCache!._cache.size, 1); +}); + +test('DNS cache works - CacheableLookup instance', withServer, async (t, _server, got) => { + const cache = new CacheableLookup(); + await t.notThrowsAsync(got('https://example.com', {dnsCache: cache, prefixUrl: ''})); - t.is(map.size, 1); + t.is((cache as any)._cache.size, 1); }); test('`isFromCache` stream property is undefined before the `response` event', withServer, async (t, server, got) => { @@ -178,3 +196,34 @@ test('`isFromCache` stream property is true if the response was cached', withSer await getStream(stream); }); + +test('can disable cache by extending the instance', withServer, async (t, server, got) => { + server.get('/', cacheEndpoint); + + const cache = new Map(); + + const instance = got.extend({cache}); + + await getStream(instance.stream('')); + const stream = instance.extend({cache: false}).stream(''); + + const response: Response = await pEvent(stream, 'response'); + t.is(response.isFromCache, false); + t.is(stream.isFromCache, false); + + await getStream(stream); +}); + +test('does not break POST requests', withServer, async (t, server, got) => { + server.post('/', async (request, response) => { + request.resume(); + response.end(JSON.stringify(request.headers)); + }); + + const headers = await got.post('', { + body: '', + cache: new Map() + }).json<{'content-length': string}>(); + + t.is(headers['content-length'], '0'); +}); diff --git a/test/cookies.ts b/test/cookies.ts index 68c5959e4..3f5abffb5 100644 --- a/test/cookies.ts +++ b/test/cookies.ts @@ -159,7 +159,7 @@ test('accepts custom `cookieJar` object', withServer, async (t, server, got) => async getCookieString(url: string) { t.is(typeof url, 'string'); - return cookies[url]; + return cookies[url] || ''; }, async setCookie(rawCookie: string, url: string) { diff --git a/test/create.ts b/test/create.ts index f0f7ad125..39b1ca763 100644 --- a/test/create.ts +++ b/test/create.ts @@ -96,15 +96,6 @@ test('extend keeps the old value if the new one is undefined', t => { ); }); -test('extend merges URL instances', t => { - // @ts-ignore Custom instance. - const a = got.extend({custom: new URL('https://example.com')}); - // @ts-ignore Custom instance. - const b = a.extend({custom: '/foo'}); - // @ts-ignore Custom instance. - t.is(b.defaults.options.custom.toString(), 'https://example.com/foo'); -}); - test('hooks are merged on got.extend()', t => { const hooksA = [() => {}]; const hooksB = [() => {}]; @@ -145,6 +136,12 @@ test('can set defaults to `got.mergeOptions(...)`', t => { }); t.true(instance.defaults.options.followRedirect); + + t.notThrows(() => { + instance.defaults.options = got.mergeOptions({}); + }); + + t.is(instance.defaults.options.followRedirect, undefined); }); test('can set mutable defaults using got.extend', t => { @@ -164,10 +161,15 @@ test('only plain objects are freezed', withServer, async (t, server, got) => { server.get('/', echoHeaders); const instance = got.extend({ - agent: new HttpAgent({keepAlive: true}) + agent: { + http: new HttpAgent({keepAlive: true}) + }, + mutableDefaults: true }); - await t.notThrowsAsync(() => instance('')); + t.notThrows(() => { + (instance.defaults.options.agent as any).http.keepAlive = true; + }); }); test('defaults are cloned on instance creation', t => { diff --git a/test/error.ts b/test/error.ts index f0ed7fc17..518960f08 100644 --- a/test/error.ts +++ b/test/error.ts @@ -2,8 +2,7 @@ import {promisify} from 'util'; import http = require('http'); import stream = require('stream'); import test from 'ava'; -import proxyquire = require('proxyquire'); -import got, {GotError, HTTPError} from '../source'; +import got, {RequestError, HTTPError} from '../source'; import withServer from './helpers/with-server'; const pStreamPipeline = promisify(stream.pipeline); @@ -34,9 +33,9 @@ test('properties', withServer, async (t, server, got) => { }); test('catches dns errors', async t => { - const error = await t.throwsAsync(got('http://doesntexist', {retry: 0})); + const error = await t.throwsAsync(got('http://doesntexist', {retry: 0})); t.truthy(error); - t.regex(error.message, /getaddrinfo ENOTFOUND/); + t.regex(error.message, /ENOTFOUND/); t.is(error.options.url.host, 'doesntexist'); t.is(error.options.method, 'GET'); }); @@ -102,11 +101,11 @@ test('contains Got options', withServer, async (t, server, got) => { response.end(); }); - const options = { + const options: {agent: false} = { agent: false }; - const error = await t.throwsAsync(got(options)); + const error = await t.throwsAsync(got(options)); t.is(error.options.agent, options.agent); }); @@ -140,10 +139,23 @@ test('`http.request` pipe error', async t => { // @ts-ignore Error tests request: () => { const proxy = new stream.PassThrough(); + + const anyProxy = proxy as any; + anyProxy.socket = { + remoteAddress: '', + prependOnceListener: () => {} + }; + + anyProxy.headers = {}; + + anyProxy.abort = () => {}; + proxy.resume(); - proxy.once('pipe', () => { + proxy.read = () => { proxy.destroy(new Error(message)); - }); + + return null; + }; return proxy; }, @@ -166,30 +178,12 @@ test('`http.request` error through CacheableRequest', async t => { }); }); -test('catches error in mimicResponse', withServer, async (t, server) => { - server.get('/', (_request, response) => { - response.end('ok'); - }); - - const mimicResponse = (): never => { - throw new Error('Error in mimic-response'); - }; - - mimicResponse['@global'] = true; - - const proxiedGot = proxyquire('../source', { - 'mimic-response': mimicResponse - }); - - await t.throwsAsync(proxiedGot(server.url), {message: 'Error in mimic-response'}); -}); - test('errors are thrown directly when options.stream is true', t => { t.throws(() => { // @ts-ignore Error tests got('https://example.com', {isStream: true, hooks: false}); }, { - message: 'Parameter `hooks` must be an Object, not boolean' + message: 'Expected value which is `predicate returns truthy for any value`, received value of type `Array`.' }); }); diff --git a/test/gzip.ts b/test/gzip.ts index c0d15ec57..2b14e35e2 100644 --- a/test/gzip.ts +++ b/test/gzip.ts @@ -72,7 +72,7 @@ test('decompress option opts out of decompressing', withServer, async (t, server response.end(gzipData); }); - const {body} = await got({decompress: false}); + const {body} = await got({decompress: false, responseType: 'buffer'}); t.is(Buffer.compare(body, gzipData), 0); }); diff --git a/test/headers.ts b/test/headers.ts index e47df3afe..da1d31ade 100644 --- a/test/headers.ts +++ b/test/headers.ts @@ -5,9 +5,10 @@ import test from 'ava'; import {Handler} from 'express'; import FormData = require('form-data'); import got, {Headers} from '../source'; -import supportsBrotli from '../source/utils/supports-brotli'; import withServer from './helpers/with-server'; +const supportsBrotli = typeof (process.versions as any).brotli === 'string'; + const echoHeaders: Handler = (request, response) => { request.resume(); response.end(JSON.stringify(request.headers)); @@ -42,10 +43,8 @@ test('does not remove user headers from `url` object argument', withServer, asyn server.get('/', echoHeaders); const headers = (await got({ - hostname: server.hostname, - port: server.port, + url: `http://${server.hostname}:${server.port}`, responseType: 'json', - protocol: 'http:', headers: { 'X-Request-Id': 'value' } @@ -123,6 +122,21 @@ test('sets `content-length` to `0` when requesting PUT with empty body', withSer t.is(headers['content-length'], '0'); }); +test('form manual `content-type` header', withServer, async (t, server, got) => { + server.post('/', echoHeaders); + + const {body} = await got.post({ + headers: { + 'content-type': 'custom' + }, + form: { + a: 1 + } + }); + const headers = JSON.parse(body); + t.is(headers['content-type'], 'custom'); +}); + test('form-data manual `content-type` header', withServer, async (t, server, got) => { server.post('/', echoHeaders); @@ -184,13 +198,13 @@ test('buffer as `options.body` sets `content-length` header', withServer, async }); test('throws on null value headers', async t => { - // @ts-ignore Error tests await t.throwsAsync(got({ + url: 'https://example.com', + // @ts-ignore Testing purposes headers: { 'user-agent': null } }), { - instanceOf: TypeError, message: 'Use `undefined` instead of `null` to delete the `user-agent` header' }); }); diff --git a/test/helpers/with-server.ts b/test/helpers/with-server.ts index 3c732d86e..c4aa3872c 100644 --- a/test/helpers/with-server.ts +++ b/test/helpers/with-server.ts @@ -4,7 +4,7 @@ import http = require('http'); import tempy = require('tempy'); import createTestServer = require('create-test-server'); import lolex = require('lolex'); -import got, {Defaults} from '../../source'; +import got, {InstanceDefaults} from '../../source'; import {ExtendedGot, ExtendedHttpServer, ExtendedTestServer, GlobalClock, InstalledClock} from './types'; export type RunTestWithServer = (t: test.ExecutionContext, server: ExtendedTestServer, got: ExtendedGot, clock: GlobalClock) => Promise | void; @@ -19,7 +19,7 @@ const generateHook = ({install}: {install?: boolean}): test.Macro<[RunTestWithSe } }) as ExtendedTestServer; - const options: Defaults = { + const options: InstanceDefaults = { // @ts-ignore Augmenting for test detection avaTest: t.title, handlers: [ diff --git a/test/hooks.ts b/test/hooks.ts index 8faa68109..e7c10a23c 100644 --- a/test/hooks.ts +++ b/test/hooks.ts @@ -1,8 +1,10 @@ +import {URL} from 'url'; import test from 'ava'; import getStream from 'get-stream'; import delay = require('delay'); import {Handler} from 'express'; -import got from '../source'; +import Responselike = require('responselike'); +import got, {RequestError} from '../source'; import withServer from './helpers/with-server'; const errorString = 'oops'; @@ -12,6 +14,10 @@ const echoHeaders: Handler = (request, response) => { response.end(JSON.stringify(request.headers)); }; +const echoUrl: Handler = (request, response) => { + response.end(request.url); +}; + const retryEndpoint: Handler = (request, response) => { if (request.headers.foo) { response.statusCode = 302; @@ -53,7 +59,45 @@ test('catches init thrown errors', async t => { throw error; }] } - }), {message: errorString}); + }), { + instanceOf: RequestError, + message: errorString + }); +}); + +test('passes init thrown errors to beforeError hooks (promise-only)', async t => { + t.plan(2); + + await t.throwsAsync(got('https://example.com', { + hooks: { + init: [() => { + throw error; + }], + beforeError: [error => { + t.is(error.message, errorString); + + return error; + }] + } + }), { + instanceOf: RequestError, + message: errorString + }); +}); + +test('passes init thrown errors to beforeError hooks (promise-only) - beforeError rejection', async t => { + const message = 'foo, bar!'; + + await t.throwsAsync(got('https://example.com', { + hooks: { + init: [() => { + throw error; + }], + beforeError: [() => { + throw new Error(message); + }] + } + }), {message}); }); test('catches beforeRequest thrown errors', async t => { @@ -63,7 +107,10 @@ test('catches beforeRequest thrown errors', async t => { throw error; }] } - }), {message: errorString}); + }), { + instanceOf: RequestError, + message: errorString + }); }); test('catches beforeRedirect thrown errors', withServer, async (t, server, got) => { @@ -76,7 +123,10 @@ test('catches beforeRedirect thrown errors', withServer, async (t, server, got) throw error; }] } - }), {message: errorString}); + }), { + instanceOf: RequestError, + message: errorString + }); }); test('catches beforeRetry thrown errors', withServer, async (t, server, got) => { @@ -89,7 +139,10 @@ test('catches beforeRetry thrown errors', withServer, async (t, server, got) => throw error; }] } - }), {message: errorString}); + }), { + instanceOf: RequestError, + message: errorString + }); }); test('catches afterResponse thrown errors', withServer, async (t, server, got) => { @@ -101,15 +154,22 @@ test('catches afterResponse thrown errors', withServer, async (t, server, got) = throw error; }] } - }), {message: errorString}); + }), { + instanceOf: RequestError, + message: errorString + }); }); -test('throws a helpful error when passing async function as init hook', async t => { - await t.throwsAsync(got('https://example.com', { +test('accepts an async function as init hook', async t => { + await got('https://example.com', { hooks: { - init: [async () => {}] + init: [ + async () => { + t.pass(); + } + ] } - }), {message: 'The `init` hook must be a synchronous function'}); + }); }); test('catches beforeRequest promise rejections', async t => { @@ -121,7 +181,10 @@ test('catches beforeRequest promise rejections', async t => { } ] } - }), {message: errorString}); + }), { + instanceOf: RequestError, + message: errorString + }); }); test('catches beforeRedirect promise rejections', withServer, async (t, server, got) => { @@ -135,7 +198,10 @@ test('catches beforeRedirect promise rejections', withServer, async (t, server, } ] } - }), {message: errorString}); + }), { + instanceOf: RequestError, + message: errorString + }); }); test('catches beforeRetry promise rejections', withServer, async (t, server, got) => { @@ -149,7 +215,10 @@ test('catches beforeRetry promise rejections', withServer, async (t, server, got } ] } - }), {message: errorString}); + }), { + instanceOf: RequestError, + message: errorString + }); }); test('catches afterResponse promise rejections', withServer, async (t, server, got) => { @@ -192,7 +261,6 @@ test('init is called with options', withServer, async (t, server, got) => { hooks: { init: [ options => { - t.is(options.url, undefined); t.is(options.context, context); } ] @@ -210,7 +278,6 @@ test('init from defaults is called with options', withServer, async (t, server, hooks: { init: [ options => { - t.is(options.url, undefined); t.is(options.context, context); } ] @@ -225,12 +292,11 @@ test('init allows modifications', withServer, async (t, server, got) => { response.end(request.headers.foo); }); - const {body} = await got('meh', { + const {body} = await got('', { headers: {}, hooks: { init: [ options => { - options.url = ''; options.headers!.foo = 'bar'; } ] @@ -271,6 +337,29 @@ test('beforeRequest allows modifications', withServer, async (t, server, got) => t.is(body.foo, 'bar'); }); +test('returning HTTP response from a beforeRequest hook', withServer, async (t, server, got) => { + server.get('/', echoUrl); + + const {statusCode, headers, body} = await got({ + hooks: { + beforeRequest: [ + () => { + return new Responselike( + 200, + {foo: 'bar'}, + Buffer.from('Hi!'), + '' + ); + } + ] + } + }); + + t.is(statusCode, 200); + t.is(headers.foo, 'bar'); + t.is(body, 'Hi!'); +}); + test('beforeRedirect is called with options and response', withServer, async (t, server, got) => { server.get('/', echoHeaders); server.get('/redirect', redirectEndpoint); @@ -579,7 +668,6 @@ test('throwing in a beforeError hook - promise', withServer, async (t, server, g response.end('ok'); }); - // @ts-ignore Error tests await t.throwsAsync(got({ hooks: { afterResponse: [ @@ -588,11 +676,11 @@ test('throwing in a beforeError hook - promise', withServer, async (t, server, g } ], beforeError: [ - () => { + (): never => { throw new Error('foobar'); }, () => { - t.fail('This shouldn\'t be called at all'); + throw new Error('This shouldn\'t be called at all'); } ] } @@ -600,7 +688,6 @@ test('throwing in a beforeError hook - promise', withServer, async (t, server, g }); test('throwing in a beforeError hook - stream', withServer, async (t, _server, got) => { - // @ts-ignore Error tests await t.throwsAsync(getStream(got.stream({ hooks: { beforeError: [ @@ -608,7 +695,7 @@ test('throwing in a beforeError hook - stream', withServer, async (t, _server, g throw new Error('foobar'); }, () => { - t.fail('This shouldn\'t be called at all'); + throw new Error('This shouldn\'t be called at all'); } ] } @@ -654,9 +741,13 @@ test('beforeError allows modifications', async t => { throw error; }, hooks: { - beforeError: [() => { - return new Error(errorString2); - }] + beforeError: [ + error => { + const newError = new Error(errorString2); + + return new RequestError(newError.message, newError, error.options); + } + ] } }), {message: errorString2}); }); @@ -669,9 +760,9 @@ test('does not break on `afterResponse` hook with JSON mode', withServer, async afterResponse: [ (response, retryWithMergedOptions) => { if (response.statusCode === 404) { - return retryWithMergedOptions({ - path: '/foobar' - }); + const url = new URL('/foobar', response.url); + + return retryWithMergedOptions({url}); } return response; @@ -688,7 +779,7 @@ test('catches HTTPErrors', withServer, async (t, _server, got) => { await t.throwsAsync(got({ hooks: { beforeError: [ - (error: Error) => { + error => { t.true(error instanceof got.HTTPError); return error; } @@ -712,3 +803,25 @@ test('timeout can be modified using a hook', withServer, async (t, server, got) retry: 0 }), {message: 'Timeout awaiting \'request\' for 500ms'}); }); + +test('beforeRequest hook is called before each request', withServer, async (t, server, got) => { + server.post('/', echoUrl); + server.post('/redirect', redirectEndpoint); + + const buffer = Buffer.from('Hello, Got!'); + let counts = 0; + + await got.post('redirect', { + body: buffer, + hooks: { + beforeRequest: [ + options => { + counts++; + t.is(options.headers['content-length'], String(buffer.length)); + } + ] + } + }); + + t.is(counts, 2); +}); diff --git a/test/http.ts b/test/http.ts index 6ca819ab0..728fdc443 100644 --- a/test/http.ts +++ b/test/http.ts @@ -1,4 +1,5 @@ import test from 'ava'; +import getStream = require('get-stream'); import got, {HTTPError, UnsupportedProtocolError} from '../source'; import withServer from './helpers/with-server'; @@ -82,6 +83,28 @@ test('custom `options.encoding`', withServer, async (t, server, got) => { t.is(data, Buffer.from(string).toString('base64')); }); +test('`options.encoding` doesn\'t affect streams', withServer, async (t, server, got) => { + const string = 'ok'; + + server.get('/', (_request, response) => { + response.end(string); + }); + + const data = await getStream(got.stream({encoding: 'base64'})); + t.is(data, string); +}); + +test('`got.stream(...).setEncoding(...)` works', withServer, async (t, server, got) => { + const string = 'ok'; + + server.get('/', (_request, response) => { + response.end(string); + }); + + const data = await getStream(got.stream('').setEncoding('base64')); + t.is(data, Buffer.from(string).toString('base64')); +}); + test('`searchParams` option', withServer, async (t, server, got) => { server.get('/', (request, response) => { t.is(request.query.recent, 'true'); @@ -92,15 +115,6 @@ test('`searchParams` option', withServer, async (t, server, got) => { t.is((await got({searchParams: 'recent=true'})).body, 'recent'); }); -test('response has `requestUrl` property even if `url` is an object', withServer, async (t, server, got) => { - server.get('/', (_request, response) => { - response.end('ok'); - }); - - t.is((await got({hostname: server.hostname, port: server.port})).requestUrl, `${server.url}/`); - t.is((await got({hostname: server.hostname, port: server.port, protocol: 'http:'})).requestUrl, `${server.url}/`); -}); - test('response contains url', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.end('ok'); @@ -114,11 +128,39 @@ test('response contains got options', withServer, async (t, server, got) => { response.end('ok'); }); - const options = { - username: 'foo' - }; + { + const options = { + username: 'foo', + password: 'bar' + }; + + const {options: normalizedOptions} = (await got(options)).request; + + t.is(normalizedOptions.username, options.username); + t.is(normalizedOptions.password, options.password); + } + + { + const options = { + username: 'foo' + }; + + const {options: normalizedOptions} = (await got(options)).request; + + t.is(normalizedOptions.username, options.username); + t.is(normalizedOptions.password, ''); + } - t.is((await got(options)).request.options.username, options.username); + { + const options = { + password: 'bar' + }; + + const {options: normalizedOptions} = (await got(options)).request; + + t.is(normalizedOptions.username, ''); + t.is(normalizedOptions.password, options.password); + } }); test('socket destroyed by the server throws ECONNRESET', withServer, async (t, server, got) => { @@ -130,3 +172,14 @@ test('socket destroyed by the server throws ECONNRESET', withServer, async (t, s code: 'ECONNRESET' }); }); + +test('the response contains timings property', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.end('ok'); + }); + + const {timings} = await got(''); + + t.truthy(timings); + t.true(timings.phases.total! >= 0); +}); diff --git a/test/https.ts b/test/https.ts index 8b7ff6419..c4e4d79ac 100644 --- a/test/https.ts +++ b/test/https.ts @@ -1,4 +1,5 @@ import test from 'ava'; +import got from '../source'; import withServer from './helpers/with-server'; test('https request without ca', withServer, async (t, server, got) => { @@ -20,3 +21,16 @@ test('https request with ca', withServer, async (t, server, got) => { }); t.is(body, 'ok'); }); + +test('http2', async t => { + const promise = got('https://httpbin.org/anything', { + http2: true + }); + + const {headers, body} = await promise; + await promise.json(); + + // @ts-ignore Pseudo headers may not be strings + t.is(headers[':status'], 200); + t.is(typeof body, 'string'); +}); diff --git a/test/options-to-url.ts b/test/options-to-url.ts deleted file mode 100644 index 1434de9db..000000000 --- a/test/options-to-url.ts +++ /dev/null @@ -1,150 +0,0 @@ -import test from 'ava'; -import is from '@sindresorhus/is'; -import optionsToUrl from '../source/utils/options-to-url'; - -const origin = 'https://google.com'; - -test('`path` and `pathname` are mutually exclusive', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({path: 'a', pathname: 'a'}); - }, { - message: 'Parameters `path` and `pathname` are mutually exclusive.' - }); -}); - -test('`path` and `search` are mutually exclusive', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({path: 'a', search: 'a'}); - }, { - message: 'Parameters `path` and `search` are mutually exclusive.' - }); -}); - -test('`path` and `searchParams` are mutually exclusive', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({path: 'a', searchParams: {}}); - }, { - message: 'Parameters `path` and `searchParams` are mutually exclusive.' - }); -}); - -test('`path` option', t => { - { - const url = optionsToUrl({origin, path: '/x?a=1'}); - t.is(url.href, `${origin}/x?a=1`); - t.true(is.urlInstance(url)); - } - - { - const url = optionsToUrl({origin, path: '/foobar'}); - t.is(url.href, `${origin}/foobar`); - t.true(is.urlInstance(url)); - } -}); - -test('`auth` is deprecated', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({auth: ''}); - }, { - message: 'Parameter `auth` is deprecated. Use `username` / `password` instead.' - }); -}); - -test('`search` and `searchParams` are mutually exclusive', t => { - t.throws(() => { - // @ts-ignore Error tests - optionsToUrl({search: 'a', searchParams: {}}); - }, { - message: 'Parameters `search` and `searchParams` are mutually exclusive.' - }); -}); - -test('`href` option', t => { - const url = optionsToUrl({href: origin}); - t.is(url.href, `${origin}/`); - t.true(is.urlInstance(url)); -}); - -test('`origin` option', t => { - const url = optionsToUrl({origin}); - t.is(url.href, `${origin}/`); - t.true(is.urlInstance(url)); -}); - -test('throws if no protocol specified', t => { - t.throws(() => { - optionsToUrl({}); - }, { - message: 'No URL protocol specified' - }); -}); - -test('`port` option', t => { - const url = optionsToUrl({origin, port: 8888}); - t.is(url.href, `${origin}:8888/`); - t.true(is.urlInstance(url)); -}); - -test('`protocol` option', t => { - const url = optionsToUrl({origin, protocol: 'http:'}); - t.is(url.href, 'http://google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`username` option', t => { - const url = optionsToUrl({origin, username: 'username'}); - t.is(url.href, 'https://username@google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`password` option', t => { - const url = optionsToUrl({origin, password: 'password'}); - t.is(url.href, 'https://:password@google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`username` option combined with `password` option', t => { - const url = optionsToUrl({origin, username: 'username', password: 'password'}); - t.is(url.href, 'https://username:password@google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`host` option', t => { - const url = optionsToUrl({protocol: 'https:', host: 'google.com'}); - t.is(url.href, 'https://google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`hostname` option', t => { - const url = optionsToUrl({protocol: 'https:', hostname: 'google.com'}); - t.is(url.href, 'https://google.com/'); - t.true(is.urlInstance(url)); -}); - -test('`pathname` option', t => { - const url = optionsToUrl({protocol: 'https:', hostname: 'google.com', pathname: '/foobar'}); - t.is(url.href, 'https://google.com/foobar'); - t.true(is.urlInstance(url)); -}); - -test('`search` option', t => { - const url = optionsToUrl({protocol: 'https:', hostname: 'google.com', search: '?a=1'}); - t.is(url.href, 'https://google.com/?a=1'); - t.true(is.urlInstance(url)); -}); - -test('`hash` option', t => { - const url = optionsToUrl({protocol: 'https:', hostname: 'google.com', hash: 'foobar'}); - t.is(url.href, 'https://google.com/#foobar'); - t.true(is.urlInstance(url)); -}); - -test('merges provided `searchParams`', t => { - const url = optionsToUrl({origin: 'https://google.com/?a=1', searchParams: {b: 2}}); - t.is(url.href, 'https://google.com/?a=1&b=2'); - t.true(is.urlInstance(url)); -}); diff --git a/test/pagination.ts b/test/pagination.ts index 71fc7c4e1..368488029 100644 --- a/test/pagination.ts +++ b/test/pagination.ts @@ -1,3 +1,4 @@ +import {URL} from 'url'; import test from 'ava'; import got, {Response} from '../source'; import withServer from './helpers/with-server'; @@ -52,12 +53,12 @@ test('retrieves all elements with JSON responseType', withServer, async (t, serv const result = await got.extend({ responseType: 'json' - }).paginate.all(''); + }).paginate.all(''); t.deepEqual(result, [1, 2]); }); -test('points to defaults when extending Got without custom `_pagination`', withServer, async (t, server, got) => { +test('points to defaults when extending Got without custom `pagination`', withServer, async (t, server, got) => { attachHandler(server, 2); const result = await got.extend().paginate.all(''); @@ -69,7 +70,7 @@ test('pagination options can be extended', withServer, async (t, server, got) => attachHandler(server, 2); const result = await got.extend({ - _pagination: { + pagination: { shouldContinue: () => false } }).paginate.all(''); @@ -81,8 +82,8 @@ test('filters elements', withServer, async (t, server, got) => { attachHandler(server, 3); const result = await got.paginate.all({ - _pagination: { - filter: (element, allItems, currentItems) => { + pagination: { + filter: (element: number, allItems: number[], currentItems: number[]) => { t.true(Array.isArray(allItems)); t.true(Array.isArray(currentItems)); @@ -98,7 +99,7 @@ test('parses elements', withServer, async (t, server, got) => { attachHandler(server, 100); const result = await got.paginate.all('?page=100', { - _pagination: { + pagination: { transform: (response: Response) => [(response as Response).body.length] } }); @@ -110,7 +111,7 @@ test('parses elements - async function', withServer, async (t, server, got) => { attachHandler(server, 100); const result = await got.paginate.all('?page=100', { - _pagination: { + pagination: { transform: async (response: Response) => [(response as Response).body.length] } }); @@ -122,13 +123,17 @@ test('custom paginate function', withServer, async (t, server, got) => { attachHandler(server, 3); const result = await got.paginate.all({ - _pagination: { - paginate: response => { - if (response.request.options.path === '/?page=3') { + pagination: { + paginate: (response: Response) => { + const url = new URL(response.url); + + if (url.search === '?page=3') { return false; } - return {path: '/?page=3'}; + url.search = '?page=3'; + + return {url}; } } }); @@ -139,9 +144,9 @@ test('custom paginate function', withServer, async (t, server, got) => { test('custom paginate function using allItems', withServer, async (t, server, got) => { attachHandler(server, 3); - const result = await got.paginate.all({ - _pagination: { - paginate: (_response, allItems) => { + const result = await got.paginate.all({ + pagination: { + paginate: (_response: Response, allItems: number[]) => { if (allItems.length === 2) { return false; } @@ -157,9 +162,9 @@ test('custom paginate function using allItems', withServer, async (t, server, go test('custom paginate function using currentItems', withServer, async (t, server, got) => { attachHandler(server, 3); - const result = await got.paginate.all({ - _pagination: { - paginate: (_response, _allItems, currentItems) => { + const result = await got.paginate.all({ + pagination: { + paginate: (_response: Response, _allItems: number[], currentItems: number[]) => { if (currentItems[0] === 3) { return false; } @@ -175,9 +180,9 @@ test('custom paginate function using currentItems', withServer, async (t, server test('iterator works', withServer, async (t, server, got) => { attachHandler(server, 5); - const results = []; + const results: number[] = []; - for await (const item of got.paginate('')) { + for await (const item of got.paginate('')) { results.push(item); } @@ -188,8 +193,8 @@ test('`shouldContinue` works', withServer, async (t, server, got) => { attachHandler(server, 2); const options = { - _pagination: { - shouldContinue: (_element: unknown, allItems: unknown[], currentItems: unknown[]) => { + pagination: { + shouldContinue: (_item: unknown, allItems: unknown[], currentItems: unknown[]) => { t.true(Array.isArray(allItems)); t.true(Array.isArray(currentItems)); @@ -211,7 +216,7 @@ test('`countLimit` works', withServer, async (t, server, got) => { attachHandler(server, 2); const options = { - _pagination: { + pagination: { countLimit: 1 } }; @@ -227,30 +232,30 @@ test('`countLimit` works', withServer, async (t, server, got) => { test('throws if no `pagination` option', async t => { const iterator = got.extend({ - _pagination: false as any + pagination: false as any }).paginate('', { prefixUrl: 'https://example.com' }); await t.throwsAsync(iterator.next(), { - message: '`options._pagination` must be implemented' + message: '`options.pagination` must be implemented' }); }); test('throws if the `pagination` option does not have `transform` property', async t => { const iterator = got.paginate('', { - _pagination: {...resetPagination}, + pagination: {...resetPagination}, prefixUrl: 'https://example.com' }); await t.throwsAsync(iterator.next(), { - message: '`options._pagination.transform` must be implemented' + message: '`options.pagination.transform` must be implemented' }); }); test('throws if the `pagination` option does not have `shouldContinue` property', async t => { const iterator = got.paginate('', { - _pagination: { + pagination: { ...resetPagination, transform: thrower }, @@ -258,28 +263,29 @@ test('throws if the `pagination` option does not have `shouldContinue` property' }); await t.throwsAsync(iterator.next(), { - message: '`options._pagination.shouldContinue` must be implemented' + message: '`options.pagination.shouldContinue` must be implemented' }); }); test('throws if the `pagination` option does not have `filter` property', async t => { const iterator = got.paginate('', { - _pagination: { + pagination: { ...resetPagination, transform: thrower, - shouldContinue: thrower + shouldContinue: thrower, + paginate: thrower }, prefixUrl: 'https://example.com' }); await t.throwsAsync(iterator.next(), { - message: '`options._pagination.filter` must be implemented' + message: '`options.pagination.filter` must be implemented' }); }); test('throws if the `pagination` option does not have `paginate` property', async t => { const iterator = got.paginate('', { - _pagination: { + pagination: { ...resetPagination, transform: thrower, shouldContinue: thrower, @@ -289,6 +295,16 @@ test('throws if the `pagination` option does not have `paginate` property', asyn }); await t.throwsAsync(iterator.next(), { - message: '`options._pagination.paginate` must be implemented' + message: '`options.pagination.paginate` must be implemented' }); }); + +test('ignores the `resolveBodyOnly` option', withServer, async (t, server, got) => { + attachHandler(server, 2); + + const items = await got.paginate.all('', { + resolveBodyOnly: true + }); + + t.deepEqual(items, [1, 2]); +}); diff --git a/test/post.ts b/test/post.ts index db3691c05..1aa6cb5d5 100644 --- a/test/post.ts +++ b/test/post.ts @@ -6,7 +6,7 @@ import delay = require('delay'); import {Handler} from 'express'; import getStream = require('get-stream'); import toReadableStream = require('to-readable-stream'); -import got from '../source'; +import got, {UploadError} from '../source'; import withServer from './helpers/with-server'; const pStreamPipeline = promisify(stream.pipeline); @@ -37,7 +37,6 @@ test('invalid body', async t => { // @ts-ignore Error tests got.post('https://example.com', {body: {}}), { - instanceOf: TypeError, message: 'The `body` option must be a stream.Readable, string or Buffer' } ); @@ -80,7 +79,6 @@ test('does NOT support sending arrays as forms', withServer, async (t, server, g await t.throwsAsync(got.post({ form: ['such', 'wow'] }), { - instanceOf: TypeError, message: 'Each query pair must be an iterable [name, value] tuple' }); }); @@ -213,7 +211,6 @@ test('`content-type` header is not overriden when object in `options.body`', wit test('throws when form body is not a plain object or array', async t => { // @ts-ignore Manual test await t.throwsAsync(got.post('https://example.com', {form: 'such=wow'}), { - instanceOf: TypeError, message: 'The `form` option must be an Object' }); }); @@ -306,3 +303,26 @@ test('catches body errors before calling pipeline() - stream', withServer, async // Wait for unhandled errors await delay(100); }); + +test('throws on upload error', withServer, async (t, server, got) => { + server.post('/', defaultEndpoint); + + const body = new stream.PassThrough(); + const message = 'oh no'; + + await t.throwsAsync(getStream(got.stream.post({ + body, + hooks: { + beforeRequest: [ + () => { + process.nextTick(() => { + body.destroy(new Error(message)); + }); + } + ] + } + })), { + instanceOf: UploadError, + message + }); +}); diff --git a/test/progress.ts b/test/progress.ts index 8088bf205..912f7cb35 100644 --- a/test/progress.ts +++ b/test/progress.ts @@ -22,17 +22,21 @@ const checkEvents: Macro<[Progress[], number?]> = (t, events, bodySize = undefin } for (const [index, event] of events.entries()) { + const isLastEvent = index === events.length - 1; + if (is.number(bodySize)) { t.is(event.percent, event.transferred / bodySize); t.true(event.percent > lastEvent.percent); + t.true(event.transferred > lastEvent.transferred); + } else if (isLastEvent) { + t.is(event.percent, 1); + t.is(event.transferred, lastEvent.transferred); + t.is(event.total, event.transferred); } else { - const isLastEvent = index === events.length - 1; - t.is(event.percent, isLastEvent ? 1 : 0); + t.is(event.percent, 0); + t.true(event.transferred > lastEvent.transferred); } - t.true(event.transferred >= lastEvent.transferred); - t.is(event.total, bodySize); - lastEvent = event; } }; @@ -199,7 +203,7 @@ test('upload progress - no body', withServer, async (t, server, got) => { { percent: 0, transferred: 0, - total: 0 + total: undefined }, { percent: 1, diff --git a/test/promise.ts b/test/promise.ts index 2e4054ce6..8e738ca0d 100644 --- a/test/promise.ts +++ b/test/promise.ts @@ -1,5 +1,4 @@ -import {ClientRequest} from 'http'; -import {Transform as TransformStream} from 'stream'; +import {ClientRequest, IncomingMessage} from 'http'; import test from 'ava'; import {Response} from '../source'; import withServer from './helpers/with-server'; @@ -22,7 +21,7 @@ test('emits response event as promise', withServer, async (t, server, got) => { }); await got('').json().on('response', (response: Response) => { - t.true(response instanceof TransformStream); + t.true(response instanceof IncomingMessage); t.true(response.readable); t.is(response.statusCode, 200); t.is(response.ip, '127.0.0.1'); diff --git a/test/redirects.ts b/test/redirects.ts index 5b74c7689..6913a8e8a 100644 --- a/test/redirects.ts +++ b/test/redirects.ts @@ -115,16 +115,6 @@ test('searchParams are not breaking redirects', withServer, async (t, server, go t.is((await got('relativeSearchParam', {searchParams: 'bang=1'})).body, 'reached'); }); -test('hostname + pathname are not breaking redirects', withServer, async (t, server, got) => { - server.get('/', reachedHandler); - server.get('/relative', relativeHandler); - - t.is((await got('relative', { - hostname: server.hostname, - pathname: '/relative' - })).body, 'reached'); -}); - test('redirects GET and HEAD requests', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.writeHead(308, { @@ -282,7 +272,7 @@ test('throws on malformed redirect URI', withServer, async (t, server, got) => { }); await t.throwsAsync(got(''), { - name: 'URIError' + message: 'URI malformed' }); }); @@ -385,7 +375,7 @@ test('body is passed on POST redirect', withServer, async (t, server, got) => { t.is(body, 'foobar'); }); -test('method overwriting can be turned off', withServer, async (t, server, got) => { +test('method rewriting can be turned off', withServer, async (t, server, got) => { server.post('/redirect', (_request, response) => { response.writeHead(302, { location: '/' @@ -411,3 +401,34 @@ test('method overwriting can be turned off', withServer, async (t, server, got) t.is(body, ''); }); + +test('clears username and password when redirecting to a different hostname', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.writeHead(302, { + location: 'https://httpbin.org/anything' + }); + response.end(); + }); + + const {headers} = await got('', { + username: 'hello', + password: 'world' + }).json(); + t.is(headers.Authorization, undefined); +}); + +test('clears the authorization header when redirecting to a different hostname', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.writeHead(302, { + location: 'https://httpbin.org/anything' + }); + response.end(); + }); + + const {headers} = await got('', { + headers: { + authorization: 'Basic aGVsbG86d29ybGQ=' + } + }).json(); + t.is(headers.Authorization, undefined); +}); diff --git a/test/response-parse.ts b/test/response-parse.ts index 7ea43eedf..5184ba078 100644 --- a/test/response-parse.ts +++ b/test/response-parse.ts @@ -101,7 +101,7 @@ test('parses non-200 responses', withServer, async (t, server, got) => { response.end(jsonResponse); }); - const error = await t.throwsAsync(got({responseType: 'json'}), {instanceOf: HTTPError}); + const error = await t.throwsAsync(got({responseType: 'json', retry: 0}), {instanceOf: HTTPError}); t.deepEqual(error.response.body, dog); }); @@ -111,7 +111,7 @@ test('ignores errors on invalid non-200 responses', withServer, async (t, server response.end('Internal error'); }); - const error = await t.throwsAsync(got({responseType: 'json'}), { + const error = await t.throwsAsync(got({responseType: 'json', retry: 0}), { instanceOf: got.HTTPError, message: 'Response code 500 (Internal Server Error)' }); diff --git a/test/retry.ts b/test/retry.ts index a48a22a39..b3a22fa29 100644 --- a/test/retry.ts +++ b/test/retry.ts @@ -22,7 +22,7 @@ const createSocketTimeoutStream = (): http.ClientRequest => { const stream = new PassThroughStream(); // @ts-ignore Mocking the behaviour of a ClientRequest stream.setTimeout = (ms, callback) => { - callback(); + process.nextTick(callback); }; // @ts-ignore Mocking the behaviour of a ClientRequest @@ -132,6 +132,7 @@ test('custom error codes', async t => { const error = await t.throwsAsync(got('https://example.com', { request: () => { const emitter = new EventEmitter() as http.ClientRequest; + emitter.abort = () => {}; emitter.end = () => {}; const error = new Error('Snap!'); @@ -204,7 +205,7 @@ test('doesn\'t retry on 413 with empty statusCodes and methods', withServer, asy const {statusCode, retryCount} = await got({ throwHttpErrors: false, retry: { - retries: 1, + limit: 1, statusCodes: [], methods: [] } @@ -219,7 +220,7 @@ test('doesn\'t retry on 413 with empty methods', withServer, async (t, server, g const {statusCode, retryCount} = await got({ throwHttpErrors: false, retry: { - retries: 1, + limit: 1, statusCodes: [413], methods: [] } diff --git a/test/socket-destroyed.ts b/test/socket-destroyed.ts deleted file mode 100644 index 63e476e81..000000000 --- a/test/socket-destroyed.ts +++ /dev/null @@ -1,23 +0,0 @@ -import test from 'ava'; -import got from '../source'; - -// TODO: Use `getActiveResources()` instead of `process.binding('timer_wrap')` when it's out: -// https://github.com/nodejs/node/pull/21453 -// eslint-disable-next-line ava/no-skip-test -test.skip('clear the progressInterval if the socket has been destroyed', async t => { - // @ts-ignore process.binding is an internal API, - // and no consensus have been made to add it to the types - // https://github.com/DefinitelyTyped/DefinitelyTyped/pull/31118 - const {Timer} = process.binding('timer_wrap'); // eslint-disable-line node/no-deprecated-api - - await t.throwsAsync(got('http://127.0.0.1:55555', {retry: 0}), { - code: 'ECONNREFUSED' - }); - - // @ts-ignore process._getActiveHandles is an internal API - const progressIntervalTimer = process._getActiveHandles().filter(handle => { - // Check if the handle is a Timer that matches the `uploadEventFrequency` interval - return handle instanceof Timer && handle._list.msecs === 150; - }); - t.is(progressIntervalTimer.length, 0); -}); diff --git a/test/stream.ts b/test/stream.ts index 18e0ded0a..74a70a566 100644 --- a/test/stream.ts +++ b/test/stream.ts @@ -9,7 +9,7 @@ import getStream = require('get-stream'); import pEvent = require('p-event'); import FormData = require('form-data'); import is from '@sindresorhus/is'; -import got from '../source'; +import got, {RequestError} from '../source'; import withServer from './helpers/with-server'; const pStreamPipeline = promisify(stream.pipeline); @@ -68,23 +68,21 @@ test('returns writeable stream', withServer, async (t, server, got) => { test('throws on write if body is specified', withServer, (t, server, got) => { server.post('/', postHandler); - t.throws(() => { - got.stream.post({body: 'wow'}).end('wow'); - }, { - message: 'Got\'s stream is not writable when the `body`, `json` or `form` option is used' - }); + const streams = [ + got.stream.post({body: 'wow'}), + got.stream.post({json: {}}), + got.stream.post({form: {}}) + ]; - t.throws(() => { - got.stream.post({json: {}}).end('wow'); - }, { - message: 'Got\'s stream is not writable when the `body`, `json` or `form` option is used' - }); + for (const stream of streams) { + t.throws(() => { + stream.end('wow'); + }, { + message: 'The payload has been already provided' + }); - t.throws(() => { - got.stream.post({form: {}}).end('wow'); - }, { - message: 'Got\'s stream is not writable when the `body`, `json` or `form` option is used' - }); + stream.destroy(); + } }); test('does not throw if using stream and passing a json option', withServer, async (t, server, got) => { @@ -102,11 +100,15 @@ test('does not throw if using stream and passing a form option', withServer, asy test('throws on write if no payload method is present', withServer, (t, server, got) => { server.post('/', postHandler); + const stream = got.stream.get(''); + t.throws(() => { - got.stream.get('').end('wow'); + stream.end('wow'); }, { - message: 'The `GET` method cannot be used with a body' + message: 'The payload has been already provided' }); + + stream.destroy(); }); test('has request event', withServer, async (t, server, got) => { @@ -317,3 +319,54 @@ test('no unhandled body stream errors', async t => { code: 'ENOTFOUND' }); }); + +test('works with pipeline', async t => { + await t.throwsAsync(pStreamPipeline( + new stream.Readable({ + read() { + this.push(null); + } + }), + got.stream.put('http://localhost:7777') + ), { + instanceOf: RequestError, + message: 'connect ECONNREFUSED 127.0.0.1:7777' + }); +}); + +test('errors have body', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.setHeader('set-cookie', 'foo=bar'); + response.end('yay'); + }); + + const error = await t.throwsAsync(getStream(got.stream('', { + cookieJar: { + setCookie: (_, __) => { + throw new Error('snap'); + }, + getCookieString: _ => { + return ''; + } + } + }))); + + t.is(error.message, 'snap'); + t.is(error.response?.body, 'yay'); +}); + +test('pipe can send modified headers', withServer, async (t, server, got) => { + server.get('/foobar', (_request, response) => { + response.setHeader('foo', 'bar'); + response.end(); + }); + + server.get('/', (_request, response) => { + got.stream('foobar').on('response', response => { + response.headers.foo = 'boo'; + }).pipe(response); + }); + + const {headers} = await got(''); + t.is(headers.foo, 'boo'); +}); diff --git a/test/timeout.ts b/test/timeout.ts index b16e49f3c..33992659c 100644 --- a/test/timeout.ts +++ b/test/timeout.ts @@ -11,7 +11,7 @@ import CacheableLookup from 'cacheable-lookup'; import {Handler} from 'express'; import pEvent = require('p-event'); import got, {TimeoutError} from '../source'; -import timedOut from '../source/utils/timed-out'; +import timedOut from '../source/core/utils/timed-out'; import slowDataStream from './helpers/slow-data-stream'; import {GlobalClock} from './helpers/types'; import withServer, {withServerAndLolex} from './helpers/with-server'; @@ -87,7 +87,7 @@ test.serial('socket timeout', async t => { const stream = new PassThroughStream(); // @ts-ignore Mocking the behaviour of a ClientRequest stream.setTimeout = (ms, callback) => { - callback(); + process.nextTick(callback); }; // @ts-ignore Mocking the behaviour of a ClientRequest @@ -133,11 +133,13 @@ test.serial('send timeout (keepalive)', withServerAndLolex, async (t, server, go response.end('ok'); }); - await got('prime', {agent: keepAliveAgent}); + await got('prime', {agent: {http: keepAliveAgent}}); await t.throwsAsync( got.post({ - agent: keepAliveAgent, + agent: { + http: keepAliveAgent + }, timeout: {send: 1}, retry: 0, body: slowDataStream(clock) @@ -198,10 +200,12 @@ test.serial('response timeout (keepalive)', withServerAndLolex, async (t, server response.end('ok'); }); - await got('prime', {agent: keepAliveAgent}); + await got('prime', {agent: {http: keepAliveAgent}}); const request = got({ - agent: keepAliveAgent, + agent: { + http: keepAliveAgent + }, timeout: {response: 1}, retry: 0 }).on('request', (request: http.ClientRequest) => { @@ -248,7 +252,8 @@ test.serial('connect timeout', withServerAndLolex, async (t, _server, got, clock test.serial('connect timeout (ip address)', withServerAndLolex, async (t, _server, got, clock) => { await t.throwsAsync( got({ - hostname: '127.0.0.1', + url: 'http://127.0.0.1', + prefixUrl: '', createConnection: options => { const socket = new net.Socket(options as object as net.SocketConstructorOpts); // @ts-ignore We know that it is readonly, but we have to test it @@ -336,7 +341,8 @@ test.serial('lookup timeout no error (ip address)', withServerAndLolex, async (t server.get('/', defaultHandler(clock)); await t.notThrowsAsync(got({ - hostname: '127.0.0.1', + url: `http://127.0.0.1:${server.port}`, + prefixUrl: '', timeout: {lookup: 1}, retry: 0 })); @@ -348,9 +354,9 @@ test.serial('lookup timeout no error (keepalive)', withServerAndLolex, async (t, response.end('ok'); }); - await got('prime', {agent: keepAliveAgent}); + await got('prime', {agent: {http: keepAliveAgent}}); await t.notThrowsAsync(got({ - agent: keepAliveAgent, + agent: {http: keepAliveAgent}, timeout: {lookup: 1}, retry: 0 }).on('request', (request: http.ClientRequest) => { @@ -358,10 +364,12 @@ test.serial('lookup timeout no error (keepalive)', withServerAndLolex, async (t, t.fail('connect event fired, invalidating test'); }); })); + + keepAliveAgent.destroy(); }); -test.serial('retries on timeout', withServerAndLolex, async (t, server, got, clock) => { - server.get('/', defaultHandler(clock)); +test.serial('retries on timeout', withServer, async (t, server, got) => { + server.get('/', () => {}); let hasTried = false; await t.throwsAsync(got({ @@ -502,8 +510,7 @@ test.serial('socket timeout is canceled on error', withServerAndLolex, async (t, timeout: {socket: 50}, retry: 0 }).on('request', (request: http.ClientRequest) => { - request.abort(); - request.emit('error', new Error(message)); + request.destroy(new Error(message)); }); await t.throwsAsync(promise, {message}); @@ -516,7 +523,7 @@ test.serial('no memory leak when using socket timeout and keepalive agent', with server.get('/', defaultHandler(clock)); const promise = got({ - agent: keepAliveAgent, + agent: {http: keepAliveAgent}, timeout: {socket: requestDelay * 2} }); @@ -530,6 +537,8 @@ test.serial('no memory leak when using socket timeout and keepalive agent', with await promise; t.is(socket.listenerCount('timeout'), 0); + + keepAliveAgent.destroy(); }); test('ensure there are no new timeouts after cancelation', t => { diff --git a/test/unix-socket.ts b/test/unix-socket.ts index 8fca64df5..ddb466937 100644 --- a/test/unix-socket.ts +++ b/test/unix-socket.ts @@ -31,7 +31,7 @@ if (process.platform !== 'win32') { }); test('throws on invalid URL', async t => { - await t.throwsAsync(got('unix:'), { + await t.throwsAsync(got('unix:', {retry: 0}), { instanceOf: got.RequestError, code: 'ENOTFOUND' }); @@ -44,4 +44,11 @@ if (process.platform !== 'win32') { const instance = got.extend({prefixUrl: url}); t.is((await instance('')).body, 'ok'); }); + + test('passes search params', withSocketServer, async (t, server) => { + server.on('/?a=1', okHandler); + + const url = format('http://unix:%s:%s', server.socketPath, '/?a=1'); + t.is((await got(url)).body, 'ok'); + }); } diff --git a/test/url-to-options.ts b/test/url-to-options.ts index 4780779eb..a9aaf320f 100644 --- a/test/url-to-options.ts +++ b/test/url-to-options.ts @@ -1,7 +1,7 @@ import url = require('url'); import {URL} from 'url'; import test from 'ava'; -import urlToOptions from '../source/utils/url-to-options'; +import urlToOptions from '../source/core/utils/url-to-options'; test('converts node legacy URL to options', t => { const exampleUrl = 'https://user:password@github.com:443/say?hello=world#bang'; diff --git a/tsconfig.json b/tsconfig.json index 26c395d01..9cc18b640 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,11 +4,13 @@ "outDir": "dist", "target": "es2018", // Node.js 10 "lib": [ - "es2018" + "es2018", + "es2019.string" ] }, "include": [ "source", - "test" + "test", + "benchmark" ] }