From 36a636dbaa510c31672871f1ba216d8f47e3760c Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Fri, 6 Jan 2023 16:20:56 +0100 Subject: [PATCH 01/20] chore: Update engines field --- packages/astro-prism/package.json | 2 +- packages/astro/astro.js | 2 +- packages/astro/package.json | 2 +- packages/create-astro/package.json | 2 +- packages/integrations/mdx/package.json | 2 +- packages/integrations/preact/package.json | 2 +- packages/integrations/react/package.json | 2 +- packages/integrations/solid/package.json | 2 +- packages/integrations/svelte/package.json | 2 +- packages/integrations/vue/package.json | 2 +- packages/telemetry/package.json | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/astro-prism/package.json b/packages/astro-prism/package.json index 938ffe757902..1de10b5951e9 100644 --- a/packages/astro-prism/package.json +++ b/packages/astro-prism/package.json @@ -35,6 +35,6 @@ "@types/prismjs": "1.26.0" }, "engines": { - "node": "^14.18.0 || >=16.12.0" + "node": ">=16.12.0" } } diff --git a/packages/astro/astro.js b/packages/astro/astro.js index 311f9cddad98..b873016bcb44 100755 --- a/packages/astro/astro.js +++ b/packages/astro/astro.js @@ -50,7 +50,7 @@ async function main() { // it's okay to hard-code the valid Node versions here since they will not change over time. if (typeof require === 'undefined') { console.error(`\nNode.js v${version} is not supported by Astro! -Please upgrade to a version of Node.js with complete ESM support: "^14.18.0 || >=16.12.0"\n`); +Please upgrade to a supported version of Node.js: ">=16.12.0"\n`); } // Not supported: Report the most helpful error message possible. diff --git a/packages/astro/package.json b/packages/astro/package.json index a087d6cf280d..feef6f7c1774 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -212,7 +212,7 @@ "unified": "^10.1.2" }, "engines": { - "node": "^14.18.0 || >=16.12.0", + "node": ">=16.12.0", "npm": ">=6.14.0" } } diff --git a/packages/create-astro/package.json b/packages/create-astro/package.json index 05e2ecebf3c2..b009db1a8865 100644 --- a/packages/create-astro/package.json +++ b/packages/create-astro/package.json @@ -54,6 +54,6 @@ "uvu": "^0.5.3" }, "engines": { - "node": "^14.18.0 || >=16.12.0" + "node": ">=16.12.0" } } diff --git a/packages/integrations/mdx/package.json b/packages/integrations/mdx/package.json index 1d3caad5e7ba..cf01b3d37f00 100644 --- a/packages/integrations/mdx/package.json +++ b/packages/integrations/mdx/package.json @@ -70,6 +70,6 @@ "vite": "^4.0.3" }, "engines": { - "node": "^14.18.0 || >=16.12.0" + "node": ">=16.12.0" } } diff --git a/packages/integrations/preact/package.json b/packages/integrations/preact/package.json index 60932161656d..acb94c320001 100644 --- a/packages/integrations/preact/package.json +++ b/packages/integrations/preact/package.json @@ -47,6 +47,6 @@ "preact": "^10.6.5" }, "engines": { - "node": "^14.18.0 || >=16.12.0" + "node": ">=16.12.0" } } diff --git a/packages/integrations/react/package.json b/packages/integrations/react/package.json index 677d0d24a0ce..4bc1c1afad5c 100644 --- a/packages/integrations/react/package.json +++ b/packages/integrations/react/package.json @@ -52,6 +52,6 @@ "@types/react-dom": "^17.0.17 || ^18.0.6" }, "engines": { - "node": "^14.18.0 || >=16.12.0" + "node": ">=16.12.0" } } diff --git a/packages/integrations/solid/package.json b/packages/integrations/solid/package.json index 94f654e8277e..334b0fb8173e 100644 --- a/packages/integrations/solid/package.json +++ b/packages/integrations/solid/package.json @@ -44,6 +44,6 @@ "solid-js": "^1.4.3" }, "engines": { - "node": "^14.18.0 || >=16.12.0" + "node": ">=16.12.0" } } diff --git a/packages/integrations/svelte/package.json b/packages/integrations/svelte/package.json index cadfb39c0cf1..f34fbe84d4ac 100644 --- a/packages/integrations/svelte/package.json +++ b/packages/integrations/svelte/package.json @@ -46,6 +46,6 @@ "svelte": "^3.54.0" }, "engines": { - "node": "^14.18.0 || >=16.12.0" + "node": ">=16.12.0" } } diff --git a/packages/integrations/vue/package.json b/packages/integrations/vue/package.json index cbff543fbc0b..2ff453ec596c 100644 --- a/packages/integrations/vue/package.json +++ b/packages/integrations/vue/package.json @@ -53,6 +53,6 @@ "vue": "^3.2.30" }, "engines": { - "node": "^14.18.0 || >=16.12.0" + "node": ">=16.12.0" } } diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index bf45c1560a3c..2830992ee43a 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -45,6 +45,6 @@ "mocha": "^9.2.2" }, "engines": { - "node": "^14.18.0 || >=16.12.0" + "node": ">=16.12.0" } } From de339fc69b4dd150a6d48710c071b62acd787da3 Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Fri, 6 Jan 2023 16:21:10 +0100 Subject: [PATCH 02/20] fix(deps): Remove node-fetch --- packages/astro/package.json | 1 - packages/integrations/node/package.json | 1 - packages/telemetry/package.json | 1 - packages/webapi/package.json | 3 +-- 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/astro/package.json b/packages/astro/package.json index feef6f7c1774..f4fbc6fb09a3 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -200,7 +200,6 @@ "eol": "^0.9.1", "memfs": "^3.4.7", "mocha": "^9.2.2", - "node-fetch": "^3.2.5", "node-mocks-http": "^1.11.0", "rehype-autolink-headings": "^6.1.1", "rehype-slug": "^5.0.1", diff --git a/packages/integrations/node/package.json b/packages/integrations/node/package.json index d7bcd67df3f3..f5e7da7efccb 100644 --- a/packages/integrations/node/package.json +++ b/packages/integrations/node/package.json @@ -37,7 +37,6 @@ "astro": "^2.0.0-beta.0" }, "devDependencies": { - "@types/node-fetch": "^2.6.2", "@types/send": "^0.17.1", "astro": "workspace:*", "astro-scripts": "workspace:*", diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index 2830992ee43a..67e0f0bb280f 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -32,7 +32,6 @@ "dset": "^3.1.2", "is-docker": "^3.0.0", "is-wsl": "^2.2.0", - "node-fetch": "^3.2.5", "which-pm-runs": "^1.1.0" }, "devDependencies": { diff --git a/packages/webapi/package.json b/packages/webapi/package.json index f4e417b47445..8bc2af2cd24d 100644 --- a/packages/webapi/package.json +++ b/packages/webapi/package.json @@ -50,8 +50,7 @@ "bugs": "https://github.com/withastro/astro/issues", "homepage": "https://github.com/withastro/astro/tree/main/packages/webapi#readme", "dependencies": { - "global-agent": "^3.0.0", - "node-fetch": "^3.2.5" + "global-agent": "^3.0.0" }, "devDependencies": { "@rollup/plugin-alias": "^3.1.9", From 01566b63b72dc622f4a6d00d4493aa5b5d765678 Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Fri, 6 Jan 2023 17:05:53 +0100 Subject: [PATCH 03/20] feat(polyfills): Remove node-fetch for undici --- packages/webapi/package.json | 3 +- packages/webapi/src/lib/fetch.ts | 102 ------------------------------- packages/webapi/src/ponyfill.ts | 2 +- pnpm-lock.yaml | 56 ++++------------- 4 files changed, 14 insertions(+), 149 deletions(-) delete mode 100644 packages/webapi/src/lib/fetch.ts diff --git a/packages/webapi/package.json b/packages/webapi/package.json index 8bc2af2cd24d..a64b31829ea4 100644 --- a/packages/webapi/package.json +++ b/packages/webapi/package.json @@ -50,7 +50,8 @@ "bugs": "https://github.com/withastro/astro/issues", "homepage": "https://github.com/withastro/astro/tree/main/packages/webapi#readme", "dependencies": { - "global-agent": "^3.0.0" + "global-agent": "^3.0.0", + "undici": "^5.14.0" }, "devDependencies": { "@rollup/plugin-alias": "^3.1.9", diff --git a/packages/webapi/src/lib/fetch.ts b/packages/webapi/src/lib/fetch.ts deleted file mode 100644 index f8500a84670a..000000000000 --- a/packages/webapi/src/lib/fetch.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { bootstrap as bootstrapGlobalAgent } from 'global-agent' -import type { RequestInit } from 'node-fetch' -import { default as nodeFetch, Headers, Request, Response } from 'node-fetch' -import Stream from 'node:stream' -import * as _ from './utils' - -bootstrapGlobalAgent({ - environmentVariableNamespace: '', -}) - -export { Headers, Request, Response } - -export const fetch = { - fetch( - resource: string | Request, - init?: Partial - ): Promise { - const resourceURL = new URL( - _.__object_isPrototypeOf(Request.prototype, resource) - ? (resource as Request).url - : _.pathToPosix(resource), - typeof Object(globalThis.process).cwd === 'function' - ? 'file:' + _.pathToPosix(process.cwd()) + '/' - : 'file:' - ) - - if (resourceURL.protocol.toLowerCase() === 'file:') { - return import('node:fs').then((fs) => { - try { - const stats = fs.statSync(resourceURL) - const body = fs.createReadStream(resourceURL) - - return new Response(body, { - status: 200, - statusText: '', - headers: { - 'content-length': String(stats.size), - date: new Date().toUTCString(), - 'last-modified': new Date(stats.mtimeMs).toUTCString(), - }, - }) - } catch (error) { - const body = new Stream.Readable() - - body._read = () => {} - body.push(null) - - return new Response(body, { - status: 404, - statusText: '', - headers: { - date: new Date().toUTCString(), - }, - }) - } - }) - } else { - return nodeFetch(resource, init) - } - }, -}.fetch - -type USVString = {} & string - -interface FetchInit { - body: RequestInit['body'] - cache: - | 'default' - | 'no-store' - | 'reload' - | 'no-cache' - | 'force-cache' - | 'only-if-cached' - credentials: 'omit' | 'same-origin' | 'include' - headers: Headers | Record - method: - | 'GET' - | 'HEAD' - | 'POST' - | 'PUT' - | 'DELETE' - | 'CONNECT' - | 'OPTIONS' - | 'TRACE' - | 'PATCH' - | USVString - mode: 'cors' | 'no-cors' | 'same-origin' | USVString - redirect: 'follow' | 'manual' | 'error' - referrer: USVString - referrerPolicy: - | 'no-referrer' - | 'no-referrer-when-downgrade' - | 'same-origin' - | 'origin' - | 'strict-origin' - | 'origin-when-cross-origin' - | 'strict-origin-when-cross-origin' - | 'unsafe-url' - integrity: USVString - keepalive: boolean - signal: AbortSignal -} diff --git a/packages/webapi/src/ponyfill.ts b/packages/webapi/src/ponyfill.ts index fc92975b5951..adf153f020b3 100644 --- a/packages/webapi/src/ponyfill.ts +++ b/packages/webapi/src/ponyfill.ts @@ -7,6 +7,7 @@ import { import { Event, EventTarget } from 'event-target-shim' import { Blob, File } from 'fetch-blob/from.js' import { FormData } from 'formdata-polyfill/esm.min.js' +import { fetch, Headers, Request, Response } from 'undici' import { URLPattern } from 'urlpattern-polyfill' import { ByteLengthQueuingStrategy, @@ -30,7 +31,6 @@ import { atob, btoa } from './lib/Base64' import { CharacterData, Comment, Text } from './lib/CharacterData' import { CustomEvent } from './lib/CustomEvent' import { DOMException } from './lib/DOMException' -import { fetch, Headers, Request, Response } from './lib/fetch' import { cancelIdleCallback, requestIdleCallback } from './lib/IdleCallback' import structuredClone from './lib/structuredClone' import { clearTimeout, setTimeout } from './lib/Timeout' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 57c8856558be..9653385ab26f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -449,7 +449,6 @@ importers: memfs: ^3.4.7 mime: ^3.0.0 mocha: ^9.2.2 - node-fetch: ^3.2.5 node-mocks-http: ^1.11.0 ora: ^6.1.0 path-browserify: ^1.0.1 @@ -575,7 +574,6 @@ importers: eol: 0.9.1 memfs: 3.4.12 mocha: 9.2.2 - node-fetch: 3.3.0 node-mocks-http: 1.12.1 rehype-autolink-headings: 6.1.1 rehype-slug: 5.1.0 @@ -1117,9 +1115,6 @@ importers: '@astrojs/node': link:../../../../integrations/node astro: link:../../.. - packages/astro/test/benchmark/simple/dist/server: - specifiers: {} - packages/astro/test/fixtures/0-css: specifiers: '@astrojs/react': workspace:* @@ -3098,7 +3093,6 @@ importers: packages/integrations/node: specifiers: '@astrojs/webapi': ^1.1.1 - '@types/node-fetch': ^2.6.2 '@types/send': ^0.17.1 astro: workspace:* astro-scripts: workspace:* @@ -3110,7 +3104,6 @@ importers: '@astrojs/webapi': link:../../webapi send: 0.18.0 devDependencies: - '@types/node-fetch': 2.6.2 '@types/send': 0.17.1 astro: link:../../astro astro-scripts: link:../../../scripts @@ -3579,7 +3572,6 @@ importers: is-docker: ^3.0.0 is-wsl: ^2.2.0 mocha: ^9.2.2 - node-fetch: ^3.2.5 which-pm-runs: ^1.1.0 dependencies: ci-info: 3.7.1 @@ -3588,7 +3580,6 @@ importers: dset: 3.1.2 is-docker: 3.0.0 is-wsl: 2.2.0 - node-fetch: 3.3.0 which-pm-runs: 1.1.0 devDependencies: '@types/debug': 4.1.7 @@ -3618,16 +3609,16 @@ importers: global-agent: ^3.0.0 magic-string: ^0.25.9 mocha: ^9.2.2 - node-fetch: ^3.2.5 rollup: ^2.79.1 rollup-plugin-terser: ^7.0.2 tslib: ^2.4.0 typescript: ~4.7.3 + undici: ^5.14.0 urlpattern-polyfill: ^1.0.0-rc5 web-streams-polyfill: ^3.2.1 dependencies: global-agent: 3.0.0 - node-fetch: 3.3.0 + undici: 5.14.0 devDependencies: '@rollup/plugin-alias': 3.1.9_rollup@2.79.1 '@rollup/plugin-inject': 4.0.4_rollup@2.79.1 @@ -7086,13 +7077,6 @@ packages: '@types/unist': 2.0.6 dev: false - /@types/node-fetch/2.6.2: - resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==} - dependencies: - '@types/node': 18.11.18 - form-data: 3.0.1 - dev: true - /@types/node/12.20.55: resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} dev: true @@ -8042,10 +8026,6 @@ packages: resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==} dev: false - /asynckit/0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: true - /at-least-node/1.0.0: resolution: {integrity: sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==} engines: {node: '>= 4.0.0'} @@ -8303,7 +8283,6 @@ packages: engines: {node: '>=10.16.0'} dependencies: streamsearch: 1.1.0 - dev: true /bytes/3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} @@ -8590,13 +8569,6 @@ packages: resolution: {integrity: sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==} dev: false - /combined-stream/1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - dependencies: - delayed-stream: 1.0.0 - dev: true - /comma-separated-tokens/2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} dev: false @@ -8806,6 +8778,7 @@ packages: /data-uri-to-buffer/4.0.0: resolution: {integrity: sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==} engines: {node: '>= 12'} + dev: false /dataloader/1.4.0: resolution: {integrity: sha512-68s5jYdlvasItOJnCuI2Q9s4q98g0pCyL3HrcKJu8KNugUl8ahgmZYg38ysLTgQjjXX3H8CJLkAvWrclWfcalw==} @@ -8958,11 +8931,6 @@ packages: slash: 4.0.0 dev: true - /delayed-stream/1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - dev: true - /delegates/1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} dev: false @@ -10105,15 +10073,6 @@ packages: resolution: {integrity: sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==} dev: true - /form-data/3.0.1: - resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} - engines: {node: '>= 6'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: true - /format/0.2.2: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} @@ -12337,6 +12296,7 @@ packages: data-uri-to-buffer: 4.0.0 fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 + dev: false /node-forge/1.3.1: resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} @@ -14288,7 +14248,6 @@ packages: /streamsearch/1.1.0: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} - dev: true /string-width/4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} @@ -14962,6 +14921,13 @@ packages: jiti: 1.16.1 dev: false + /undici/5.14.0: + resolution: {integrity: sha512-yJlHYw6yXPPsuOH0x2Ib1Km61vu4hLiRRQoafs+WUgX1vO64vgnxiCEN9dpIrhZyHFsai3F0AEj4P9zy19enEQ==} + engines: {node: '>=12.18'} + dependencies: + busboy: 1.6.0 + dev: false + /undici/5.9.1: resolution: {integrity: sha512-6fB3a+SNnWEm4CJbgo0/CWR8RGcOCQP68SF4X0mxtYTq2VNN8T88NYrWVBAeSX+zb7bny2dx2iYhP3XHi00omg==} engines: {node: '>=12.18'} From 369cd6a205c9b32e9752b7e32a6ab25a919cc5cf Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Mon, 9 Jan 2023 14:20:49 +0100 Subject: [PATCH 04/20] feat(webapi): Remove node-fetch from the webapis polyfills for undici --- packages/webapi/LICENSE | 2 -- packages/webapi/README.md | 2 -- packages/webapi/run/build.js | 16 ++++----- packages/webapi/src/ponyfill.ts | 7 +++- packages/webapi/src/types.d.ts | 1 - packages/webapi/test/fetch.js | 60 --------------------------------- 6 files changed, 14 insertions(+), 74 deletions(-) diff --git a/packages/webapi/LICENSE b/packages/webapi/LICENSE index 9dda027eb4b2..11ffac14c040 100644 --- a/packages/webapi/LICENSE +++ b/packages/webapi/LICENSE @@ -32,6 +32,4 @@ Code from [fetch-blob](https://www.npmjs.com/package/fetch-blob) is licensed und Code from [formdata-polyfill](https://www.npmjs.com/package/formdata-polyfill) is licensed under the MIT License (MIT), Copyright Jimmy Wärting. -Code from [node-fetch](https://www.npmjs.com/package/node-fetch) is licensed under the MIT License (MIT), Copyright Node Fetch Team. - Code from [web-streams-polyfill](https://www.npmjs.com/package/web-streams-polyfill) is licensed under the MIT License (MIT), Copyright Mattias Buelens and Diwank Singh Tomer. diff --git a/packages/webapi/README.md b/packages/webapi/README.md index 8e1c66280444..adb31b26e3d3 100644 --- a/packages/webapi/README.md +++ b/packages/webapi/README.md @@ -174,6 +174,4 @@ Code from [fetch-blob](https://www.npmjs.com/package/fetch-blob) is licensed und Code from [formdata-polyfill](https://www.npmjs.com/package/formdata-polyfill) is licensed under the MIT License (MIT), Copyright Jimmy Wärting. -Code from [node-fetch](https://www.npmjs.com/package/node-fetch) is licensed under the MIT License (MIT), Copyright Node Fetch Team. - Code from [web-streams-polyfill](https://www.npmjs.com/package/web-streams-polyfill) is licensed under the MIT License (MIT), Copyright Mattias Buelens and Diwank Singh Tomer. diff --git a/packages/webapi/run/build.js b/packages/webapi/run/build.js index 63e17e84ce06..cfb58f69745d 100644 --- a/packages/webapi/run/build.js +++ b/packages/webapi/run/build.js @@ -1,17 +1,17 @@ -import { rollup } from 'rollup' +import { default as alias } from '@rollup/plugin-alias' +import { default as inject } from '@rollup/plugin-inject' import { nodeResolve } from '@rollup/plugin-node-resolve' -import path from 'node:path' -import { createRequire } from 'node:module' +import { default as typescript } from '@rollup/plugin-typescript' +import { default as MagicString } from 'magic-string' import { readFile as nodeReadFile, rename, rm, writeFile, } from 'node:fs/promises' -import { default as MagicString } from 'magic-string' -import { default as alias } from '@rollup/plugin-alias' -import { default as inject } from '@rollup/plugin-inject' -import { default as typescript } from '@rollup/plugin-typescript' +import { createRequire } from 'node:module' +import path from 'node:path' +import { rollup } from 'rollup' const readFileCache = Object.create(null) const require = createRequire(import.meta.url) @@ -178,7 +178,7 @@ async function build() { inputOptions: { input: 'src/polyfill.ts', plugins: plugins, - external: ['node-fetch', 'global-agent'], + external: ['undici', 'global-agent'], onwarn(warning, warn) { if (warning.code !== 'UNRESOLVED_IMPORT') warn(warning) }, diff --git a/packages/webapi/src/ponyfill.ts b/packages/webapi/src/ponyfill.ts index adf153f020b3..671b036e8b20 100644 --- a/packages/webapi/src/ponyfill.ts +++ b/packages/webapi/src/ponyfill.ts @@ -7,7 +7,7 @@ import { import { Event, EventTarget } from 'event-target-shim' import { Blob, File } from 'fetch-blob/from.js' import { FormData } from 'formdata-polyfill/esm.min.js' -import { fetch, Headers, Request, Response } from 'undici' +import * as undici from 'undici' import { URLPattern } from 'urlpattern-polyfill' import { ByteLengthQueuingStrategy, @@ -83,6 +83,11 @@ import { initPromise } from './lib/Promise' import { initRelativeIndexingMethod } from './lib/RelativeIndexingMethod' import { initString } from './lib/String' +const fetch = undici.fetch +const Headers = undici.Headers +const Response = undici.Response +const Request = undici.Request + export { AbortController, AbortSignal, diff --git a/packages/webapi/src/types.d.ts b/packages/webapi/src/types.d.ts index 09c57387bcf2..20db0d677c67 100644 --- a/packages/webapi/src/types.d.ts +++ b/packages/webapi/src/types.d.ts @@ -3,5 +3,4 @@ declare module '@ungap/structured-clone/esm/index.js' declare module '@ungap/structured-clone/esm/deserialize.js' declare module '@ungap/structured-clone/esm/serialize.js' declare module 'abort-controller/dist/abort-controller.mjs' -declare module 'node-fetch/src/index.js' declare module 'web-streams-polyfill/dist/ponyfill.es6.mjs' diff --git a/packages/webapi/test/fetch.js b/packages/webapi/test/fetch.js index ae5ae038649b..49aab31dc2d0 100644 --- a/packages/webapi/test/fetch.js +++ b/packages/webapi/test/fetch.js @@ -22,66 +22,6 @@ describe('Fetch', () => { expect(json).to.be.an('array') }) - it('Fetch with file', async () => { - const { fetch } = target - - const url = new URL('../package.json', import.meta.url) - - const response = await fetch(url) - - expect(response.constructor).to.equal(target.Response) - - expect(response.status).to.equal(200) - expect(response.statusText).to.be.empty - expect(response.headers.has('date')).to.equal(true) - expect(response.headers.has('content-length')).to.equal(true) - expect(response.headers.has('last-modified')).to.equal(true) - - const json = await response.json() - - expect(json.name).to.equal('@astrojs/webapi') - }) - - it('Fetch with missing file', async () => { - const { fetch } = target - - const url = new URL('../missing.json', import.meta.url) - - const response = await fetch(url) - - expect(response.constructor).to.equal(target.Response) - - expect(response.status).to.equal(404) - expect(response.statusText).to.be.empty - expect(response.headers.has('date')).to.equal(true) - expect(response.headers.has('content-length')).to.equal(false) - expect(response.headers.has('last-modified')).to.equal(false) - }) - - it('Fetch with (file) Request', async () => { - const { Request, fetch } = target - - const request = new Request(new URL('../package.json', import.meta.url)) - - const response = await fetch(request) - - expect(response.constructor).to.equal(target.Response) - - const json = await response.json() - - expect(json.name).to.equal('@astrojs/webapi') - }) - - it('Fetch with relative file', async () => { - const { fetch } = target - - const response = await fetch('package.json') - - const json = await response.json() - - expect(json.name).to.equal('@astrojs/webapi') - }) - it('Fetch with data', async () => { const { fetch } = target From 968f45cc31033615e6cfb8c17c770f5286159b04 Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Mon, 9 Jan 2023 14:22:04 +0100 Subject: [PATCH 05/20] feat(core): Remove node-fetch for undici in Astro core --- packages/astro/package.json | 1 + packages/astro/src/runtime/server/response.ts | 23 +++++++++++++++---- packages/astro/test/ssr-api-route.test.js | 4 ++-- packages/astro/test/test-utils.js | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/astro/package.json b/packages/astro/package.json index f4fbc6fb09a3..2021209cf99d 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -208,6 +208,7 @@ "rollup": "^3.9.0", "sass": "^1.52.2", "srcset-parse": "^1.1.0", + "undici": "^5.14.0", "unified": "^10.1.2" }, "engines": { diff --git a/packages/astro/src/runtime/server/response.ts b/packages/astro/src/runtime/server/response.ts index ae374d1aa14e..5d5800debed9 100644 --- a/packages/astro/src/runtime/server/response.ts +++ b/packages/astro/src/runtime/server/response.ts @@ -3,6 +3,21 @@ const isNodeJS = let StreamingCompatibleResponse: typeof Response | undefined; +// Undici on Node 16 does not support simply iterating over a ReadableStream +async function* streamAsyncIterator(stream: ReadableStream) { + const reader = stream.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) return; + yield value; + } + } finally { + reader.releaseLock(); + } +} + function createResponseClass() { StreamingCompatibleResponse = class extends Response { #isStream: boolean; @@ -21,9 +36,9 @@ function createResponseClass() { async text(): Promise { if (this.#isStream && isNodeJS) { let decoder = new TextDecoder(); - let body = this.#body as AsyncIterable; + let body = this.#body; let out = ''; - for await (let chunk of body) { + for await (let chunk of streamAsyncIterator(body)) { out += decoder.decode(chunk); } return out; @@ -33,10 +48,10 @@ function createResponseClass() { async arrayBuffer(): Promise { if (this.#isStream && isNodeJS) { - let body = this.#body as AsyncIterable; + let body = this.#body; let chunks: Uint8Array[] = []; let len = 0; - for await (let chunk of body) { + for await (let chunk of streamAsyncIterator(body)) { chunks.push(chunk); len += chunk.length; } diff --git a/packages/astro/test/ssr-api-route.test.js b/packages/astro/test/ssr-api-route.test.js index 33b1ffdabcf7..cafbdf32c70a 100644 --- a/packages/astro/test/ssr-api-route.test.js +++ b/packages/astro/test/ssr-api-route.test.js @@ -1,7 +1,7 @@ import { expect } from 'chai'; -import { loadFixture } from './test-utils.js'; +import { File, FormData } from 'undici'; import testAdapter from './test-adapter.js'; -import { FormData, File } from 'node-fetch'; +import { loadFixture } from './test-utils.js'; describe('API routes in SSR', () => { /** @type {import('./test-utils').Fixture} */ diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index 8c12799f15a1..905913512284 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -19,7 +19,7 @@ polyfill(globalThis, { }); /** - * @typedef {import('node-fetch').Response} Response + * @typedef {import('undici').Response} Response * @typedef {import('../src/core/dev/dev').DedvServer} DevServer * @typedef {import('../src/@types/astro').AstroConfig} AstroConfig * @typedef {import('../src/core/preview/index').PreviewServer} PreviewServer From 51a45dead6c8c5d0f967f97979f3cfe049260f3a Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Mon, 9 Jan 2023 14:22:21 +0100 Subject: [PATCH 06/20] feat(telemetry): Remove node-fetch for undici --- packages/telemetry/package.json | 1 + packages/telemetry/src/post.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index 67e0f0bb280f..32bd2a1d1e77 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -32,6 +32,7 @@ "dset": "^3.1.2", "is-docker": "^3.0.0", "is-wsl": "^2.2.0", + "undici": "^5.14.0", "which-pm-runs": "^1.1.0" }, "devDependencies": { diff --git a/packages/telemetry/src/post.ts b/packages/telemetry/src/post.ts index a0647075f988..4ce22738843b 100644 --- a/packages/telemetry/src/post.ts +++ b/packages/telemetry/src/post.ts @@ -1,4 +1,4 @@ -import fetch from 'node-fetch'; +import { fetch } from 'undici'; const ASTRO_TELEMETRY_ENDPOINT = `https://telemetry.astro.build/api/v1/record`; From 406ec02c1762c6a0144efc54bac592b9a003021e Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Mon, 9 Jan 2023 14:22:49 +0100 Subject: [PATCH 07/20] feat(node): Remove node-fetch for undici in node integration --- packages/integrations/node/package.json | 3 ++- packages/integrations/node/src/response-iterator.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/integrations/node/package.json b/packages/integrations/node/package.json index f5e7da7efccb..5b9176c29235 100644 --- a/packages/integrations/node/package.json +++ b/packages/integrations/node/package.json @@ -42,6 +42,7 @@ "astro-scripts": "workspace:*", "chai": "^4.3.6", "mocha": "^9.2.2", - "node-mocks-http": "^1.11.0" + "node-mocks-http": "^1.11.0", + "undici": "^5.14.0" } } diff --git a/packages/integrations/node/src/response-iterator.ts b/packages/integrations/node/src/response-iterator.ts index 7700e9331e92..becd8be1b505 100644 --- a/packages/integrations/node/src/response-iterator.ts +++ b/packages/integrations/node/src/response-iterator.ts @@ -4,7 +4,7 @@ * - https://github.com/apollographql/apollo-client/blob/main/src/utilities/common/responseIterator.ts */ -import type { Response as NodeResponse } from 'node-fetch'; +import type { Response as NodeResponse } from 'undici'; import { Readable as NodeReadableStream } from 'stream'; interface NodeStreamIterator { From 7cee1300b2cd1a70e85c2018fcde795af0e414f8 Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Mon, 9 Jan 2023 14:23:14 +0100 Subject: [PATCH 08/20] feat(vercel): Remove node-fetch for undici in Vercel integration --- .../vercel/src/serverless/entrypoint.ts | 11 +- .../node18.ts => request-transform.ts} | 0 .../serverless/request-transform/legacy.ts | 111 ------------------ 3 files changed, 1 insertion(+), 121 deletions(-) rename packages/integrations/vercel/src/serverless/{request-transform/node18.ts => request-transform.ts} (100%) delete mode 100644 packages/integrations/vercel/src/serverless/request-transform/legacy.ts diff --git a/packages/integrations/vercel/src/serverless/entrypoint.ts b/packages/integrations/vercel/src/serverless/entrypoint.ts index daa811015b86..71ad2bfaef7f 100644 --- a/packages/integrations/vercel/src/serverless/entrypoint.ts +++ b/packages/integrations/vercel/src/serverless/entrypoint.ts @@ -3,21 +3,12 @@ import type { SSRManifest } from 'astro'; import { App } from 'astro/app'; import type { IncomingMessage, ServerResponse } from 'node:http'; -import * as requestTransformLegacy from './request-transform/legacy.js'; -import * as requestTransformNode18 from './request-transform/node18.js'; +import { getRequest, setResponse } from './request-transform'; polyfill(globalThis, { exclude: 'window document', }); -// Node 18+ has a new API for request/response, while older versions use node-fetch -// When we drop support for Node 14, we can remove the legacy code by switching to undici - -const nodeVersion = parseInt(process.version.split('.')[0].slice(1)); // 'v14.17.0' -> 14 - -const { getRequest, setResponse } = - nodeVersion >= 18 ? requestTransformNode18 : requestTransformLegacy; - export const createExports = (manifest: SSRManifest) => { const app = new App(manifest); diff --git a/packages/integrations/vercel/src/serverless/request-transform/node18.ts b/packages/integrations/vercel/src/serverless/request-transform.ts similarity index 100% rename from packages/integrations/vercel/src/serverless/request-transform/node18.ts rename to packages/integrations/vercel/src/serverless/request-transform.ts diff --git a/packages/integrations/vercel/src/serverless/request-transform/legacy.ts b/packages/integrations/vercel/src/serverless/request-transform/legacy.ts deleted file mode 100644 index 7212431c72a0..000000000000 --- a/packages/integrations/vercel/src/serverless/request-transform/legacy.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { App } from 'astro/app'; -import type { IncomingMessage, ServerResponse } from 'node:http'; -import { Readable } from 'node:stream'; - -const clientAddressSymbol = Symbol.for('astro.clientAddress'); - -/* - Credits to the SvelteKit team - https://github.com/sveltejs/kit/blob/69913e9fda054fa6a62a80e2bb4ee7dca1005796/packages/kit/src/node.js -*/ - -function get_raw_body(req: IncomingMessage) { - return new Promise((fulfil, reject) => { - const h = req.headers; - - if (!h['content-type']) { - return fulfil(null); - } - - req.on('error', reject); - - const length = Number(h['content-length']); - - // https://github.com/jshttp/type-is/blob/c1f4388c71c8a01f79934e68f630ca4a15fffcd6/index.js#L81-L95 - if (isNaN(length) && h['transfer-encoding'] == null) { - return fulfil(null); - } - - let data = new Uint8Array(length || 0); - - if (length > 0) { - let offset = 0; - req.on('data', (chunk) => { - const new_len = offset + Buffer.byteLength(chunk); - - if (new_len > length) { - return reject({ - status: 413, - reason: 'Exceeded "Content-Length" limit', - }); - } - - data.set(chunk, offset); - offset = new_len; - }); - } else { - req.on('data', (chunk) => { - const new_data = new Uint8Array(data.length + chunk.length); - new_data.set(data, 0); - new_data.set(chunk, data.length); - data = new_data; - }); - } - - req.on('end', () => { - fulfil(data); - }); - }); -} - -export async function getRequest(base: string, req: IncomingMessage): Promise { - let headers = req.headers as Record; - if (req.httpVersionMajor === 2) { - // we need to strip out the HTTP/2 pseudo-headers because node-fetch's - // Request implementation doesn't like them - headers = Object.assign({}, headers); - delete headers[':method']; - delete headers[':path']; - delete headers[':authority']; - delete headers[':scheme']; - } - const request = new Request(base + req.url, { - method: req.method, - headers, - body: await get_raw_body(req), // TODO stream rather than buffer - }); - Reflect.set(request, clientAddressSymbol, headers['x-forwarded-for']); - return request; -} - -export async function setResponse( - app: App, - res: ServerResponse, - response: Response -): Promise { - const headers = Object.fromEntries(response.headers); - - if (response.headers.has('set-cookie')) { - // @ts-expect-error (headers.raw() is non-standard) - headers['set-cookie'] = response.headers.raw()['set-cookie']; - } - - if (app.setCookieHeaders) { - const setCookieHeaders: Array = Array.from(app.setCookieHeaders(response)); - if (setCookieHeaders.length) { - res.setHeader('Set-Cookie', setCookieHeaders); - } - } - - res.writeHead(response.status, headers); - - if (response.body instanceof Readable) { - response.body.pipe(res); - } else { - if (response.body) { - res.write(await response.arrayBuffer()); - } - - res.end(); - } -} From d7aadc67b6c0bb2425d5ef7cbb2bd2e566f37f5e Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Mon, 9 Jan 2023 14:23:26 +0100 Subject: [PATCH 09/20] chore: update lockfile --- pnpm-lock.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9653385ab26f..f16b6e01c4f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -476,6 +476,7 @@ importers: supports-esm: ^1.0.0 tsconfig-resolver: ^3.0.1 typescript: '*' + undici: ^5.14.0 unified: ^10.1.2 unist-util-visit: ^4.1.0 vfile: ^5.3.2 @@ -582,6 +583,7 @@ importers: rollup: 3.9.1 sass: 1.57.1 srcset-parse: 1.1.0 + undici: 5.14.0 unified: 10.1.2 packages/astro-prism: @@ -3100,6 +3102,7 @@ importers: mocha: ^9.2.2 node-mocks-http: ^1.11.0 send: ^0.18.0 + undici: ^5.14.0 dependencies: '@astrojs/webapi': link:../../webapi send: 0.18.0 @@ -3110,6 +3113,7 @@ importers: chai: 4.3.7 mocha: 9.2.2 node-mocks-http: 1.12.1 + undici: 5.14.0 packages/integrations/node/test/fixtures/api-route: specifiers: @@ -3572,6 +3576,7 @@ importers: is-docker: ^3.0.0 is-wsl: ^2.2.0 mocha: ^9.2.2 + undici: ^5.14.0 which-pm-runs: ^1.1.0 dependencies: ci-info: 3.7.1 @@ -3580,6 +3585,7 @@ importers: dset: 3.1.2 is-docker: 3.0.0 is-wsl: 2.2.0 + undici: 5.14.0 which-pm-runs: 1.1.0 devDependencies: '@types/debug': 4.1.7 @@ -14926,7 +14932,6 @@ packages: engines: {node: '>=12.18'} dependencies: busboy: 1.6.0 - dev: false /undici/5.9.1: resolution: {integrity: sha512-6fB3a+SNnWEm4CJbgo0/CWR8RGcOCQP68SF4X0mxtYTq2VNN8T88NYrWVBAeSX+zb7bny2dx2iYhP3XHi00omg==} From 06af09b19b461e8546f08fac2a72957f419b23e7 Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Mon, 9 Jan 2023 14:26:14 +0100 Subject: [PATCH 10/20] chore: update lockfile --- pnpm-lock.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5206bc06293b..e66180e7b9d8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10051,6 +10051,11 @@ packages: resolution: {integrity: sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==} dev: true + /for-each/0.3.3: + resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} + dependencies: + is-callable: 1.2.7 + /format/0.2.2: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} From 07b69c59b646d862fcf6ceb56be094e09dbccd5e Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Mon, 9 Jan 2023 15:34:28 +0100 Subject: [PATCH 11/20] chore: changeset --- .changeset/curvy-beds-warn.md | 16 ++++++++++++++++ .changeset/stupid-wolves-explain.md | 7 +++++++ 2 files changed, 23 insertions(+) create mode 100644 .changeset/curvy-beds-warn.md create mode 100644 .changeset/stupid-wolves-explain.md diff --git a/.changeset/curvy-beds-warn.md b/.changeset/curvy-beds-warn.md new file mode 100644 index 000000000000..a09fb9a8176c --- /dev/null +++ b/.changeset/curvy-beds-warn.md @@ -0,0 +1,16 @@ +--- +'astro': major +'@astrojs/prism': major +'create-astro': major +'@astrojs/mdx': minor +'@astrojs/node': major +'@astrojs/preact': major +'@astrojs/react': major +'@astrojs/solid-js': major +'@astrojs/svelte': major +'@astrojs/vercel': major +'@astrojs/vue': major +'@astrojs/telemetry': major +--- + +Remove support for Node 14. Minimum supported Node version is now >=16.12.0 diff --git a/.changeset/stupid-wolves-explain.md b/.changeset/stupid-wolves-explain.md new file mode 100644 index 000000000000..f6597b087a09 --- /dev/null +++ b/.changeset/stupid-wolves-explain.md @@ -0,0 +1,7 @@ +--- +'@astrojs/webapi': major +--- + +Replace node-fetch's polyfill with undici. + +Since `undici` does not support it, this changes remove support for the `file:` protocol From d1e6f1fef42e4e67bd20c77baa9990b21abde595 Mon Sep 17 00:00:00 2001 From: Princesseuh Date: Mon, 9 Jan 2023 17:13:29 +0100 Subject: [PATCH 12/20] fix(set): Fix set directives not streaming correctly on Node 16 --- packages/astro/src/runtime/server/escape.ts | 17 ++++++++++++----- packages/astro/src/runtime/server/response.ts | 17 ++--------------- packages/astro/src/runtime/server/util.ts | 14 ++++++++++++++ packages/astro/test/streaming.test.js | 10 +++++----- packages/astro/test/test-utils.js | 14 ++++++++++++++ 5 files changed, 47 insertions(+), 25 deletions(-) diff --git a/packages/astro/src/runtime/server/escape.ts b/packages/astro/src/runtime/server/escape.ts index 48041a96b0d9..818c3a52f11b 100644 --- a/packages/astro/src/runtime/server/escape.ts +++ b/packages/astro/src/runtime/server/escape.ts @@ -1,4 +1,5 @@ import { escape } from 'html-escaper'; +import { streamAsyncIterator } from './util.js'; // Leverage the battle-tested `html-escaper` npm package. export const escapeHTML = escape; @@ -58,9 +59,15 @@ export function isHTMLBytes(value: any): value is HTMLBytes { return Object.prototype.toString.call(value) === '[object HTMLBytes]'; } -async function* unescapeChunksAsync(iterable: AsyncIterable): any { - for await (const chunk of iterable) { - yield unescapeHTML(chunk as BlessedType); +async function* unescapeChunksAsync(iterable: ReadableStream | string): any { + if (iterable instanceof ReadableStream) { + for await (const chunk of streamAsyncIterator(iterable)) { + yield unescapeHTML(chunk as BlessedType); + } + } else { + for await (const chunk of iterable) { + yield unescapeHTML(chunk as BlessedType); + } } } @@ -82,7 +89,7 @@ export function unescapeHTML( } // If a response, stream out the chunks else if (str instanceof Response && str.body) { - const body = str.body as unknown as AsyncIterable; + const body = str.body; return unescapeChunksAsync(body); } // If a promise, await the result and mark that. @@ -92,7 +99,7 @@ export function unescapeHTML( }); } else if (Symbol.iterator in str) { return unescapeChunks(str); - } else if (Symbol.asyncIterator in str) { + } else if (Symbol.asyncIterator in str || str instanceof ReadableStream) { return unescapeChunksAsync(str); } } diff --git a/packages/astro/src/runtime/server/response.ts b/packages/astro/src/runtime/server/response.ts index 5d5800debed9..a39ceaa439d0 100644 --- a/packages/astro/src/runtime/server/response.ts +++ b/packages/astro/src/runtime/server/response.ts @@ -1,23 +1,10 @@ +import { streamAsyncIterator } from './util.js'; + const isNodeJS = typeof process === 'object' && Object.prototype.toString.call(process) === '[object process]'; let StreamingCompatibleResponse: typeof Response | undefined; -// Undici on Node 16 does not support simply iterating over a ReadableStream -async function* streamAsyncIterator(stream: ReadableStream) { - const reader = stream.getReader(); - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) return; - yield value; - } - } finally { - reader.releaseLock(); - } -} - function createResponseClass() { StreamingCompatibleResponse = class extends Response { #isStream: boolean; diff --git a/packages/astro/src/runtime/server/util.ts b/packages/astro/src/runtime/server/util.ts index 9f0fdbec253f..b38fe5ef1b9f 100644 --- a/packages/astro/src/runtime/server/util.ts +++ b/packages/astro/src/runtime/server/util.ts @@ -31,3 +31,17 @@ export function serializeListValue(value: any) { export function isPromise(value: any): value is Promise { return !!value && typeof value === 'object' && typeof value.then === 'function'; } + +export async function* streamAsyncIterator(stream: ReadableStream) { + const reader = stream.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) return; + yield value; + } + } finally { + reader.releaseLock(); + } +} diff --git a/packages/astro/test/streaming.test.js b/packages/astro/test/streaming.test.js index bc50c031818e..47dedac22a47 100644 --- a/packages/astro/test/streaming.test.js +++ b/packages/astro/test/streaming.test.js @@ -1,7 +1,7 @@ -import { isWindows, loadFixture } from './test-utils.js'; import { expect } from 'chai'; -import testAdapter from './test-adapter.js'; import * as cheerio from 'cheerio'; +import testAdapter from './test-adapter.js'; +import { isWindows, loadFixture, streamAsyncIterator } from './test-utils.js'; describe('Streaming', () => { if (isWindows) return; @@ -32,7 +32,7 @@ describe('Streaming', () => { it('Body is chunked', async () => { let res = await fixture.fetch('/'); let chunks = []; - for await (const bytes of res.body) { + for await (const bytes of streamAsyncIterator(res.body)) { let chunk = bytes.toString('utf-8'); chunks.push(chunk); } @@ -61,7 +61,7 @@ describe('Streaming', () => { const response = await app.render(request); let chunks = []; let decoder = new TextDecoder(); - for await (const bytes of response.body) { + for await (const bytes of streamAsyncIterator(response.body)) { let chunk = decoder.decode(bytes); chunks.push(chunk); } @@ -102,7 +102,7 @@ describe('Streaming disabled', () => { it('Body is chunked', async () => { let res = await fixture.fetch('/'); let chunks = []; - for await (const bytes of res.body) { + for await (const bytes of streamAsyncIterator(res.body)) { let chunk = bytes.toString('utf-8'); chunks.push(chunk); } diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index 41f7504ca6ab..27e4caa5ea9c 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -303,3 +303,17 @@ export const isWindows = os.platform() === 'win32'; export function fixLineEndings(str) { return str.replace(/\r\n/g, '\n'); } + +export async function* streamAsyncIterator(stream) { + const reader = stream.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) return; + yield value; + } + } finally { + reader.releaseLock(); + } +} From 1e54efa00aac3879517ccba017618a7e7fbe8ead Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Mon, 9 Jan 2023 13:52:21 -0500 Subject: [PATCH 13/20] Try another approach --- packages/astro/src/runtime/server/escape.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/astro/src/runtime/server/escape.ts b/packages/astro/src/runtime/server/escape.ts index 818c3a52f11b..879f1e75bfb9 100644 --- a/packages/astro/src/runtime/server/escape.ts +++ b/packages/astro/src/runtime/server/escape.ts @@ -59,8 +59,12 @@ export function isHTMLBytes(value: any): value is HTMLBytes { return Object.prototype.toString.call(value) === '[object HTMLBytes]'; } +function hasGetReader(obj: unknown): obj is ReadableStream { + return typeof (obj as any).getReader === 'function'; +} + async function* unescapeChunksAsync(iterable: ReadableStream | string): any { - if (iterable instanceof ReadableStream) { + if (hasGetReader(iterable)) { for await (const chunk of streamAsyncIterator(iterable)) { yield unescapeHTML(chunk as BlessedType); } @@ -99,7 +103,7 @@ export function unescapeHTML( }); } else if (Symbol.iterator in str) { return unescapeChunks(str); - } else if (Symbol.asyncIterator in str || str instanceof ReadableStream) { + } else if (Symbol.asyncIterator in str || hasGetReader(str)) { return unescapeChunksAsync(str); } } From 283dacfa0e5e4aaeace213d59d2b7066b3ce6418 Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Mon, 9 Jan 2023 14:01:19 -0500 Subject: [PATCH 14/20] Debugging --- packages/astro/test/test-utils.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index 27e4caa5ea9c..2a9ea679fd78 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -305,6 +305,10 @@ export function fixLineEndings(str) { } export async function* streamAsyncIterator(stream) { + if(typeof stream.getReader !== 'function') { + console.log(stream); + throw new Error(`stream.getReader() is not a function.`); + } const reader = stream.getReader(); try { From a8eeec493486e84c04fb1c7427857a9e6cfa45fb Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Mon, 9 Jan 2023 14:18:54 -0500 Subject: [PATCH 15/20] Debug fetch --- packages/astro/test/test-utils.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index 2a9ea679fd78..f0d705cb58c7 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -157,7 +157,10 @@ export async function loadFixture(inlineConfig) { }, config, resolveUrl, - fetch: (url, init) => fetch(resolveUrl(url), init), + fetch: (url, init) => { + console.log('fetch2', fetch) + return fetch(resolveUrl(url), init) + }, preview: async (opts = {}) => { process.env.NODE_ENV = 'production'; const previewServer = await preview(settings, { logging, telemetry, ...opts }); From ff698f455543d23d2f64e588f1fa2b3069d477e7 Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Mon, 9 Jan 2023 14:23:27 -0500 Subject: [PATCH 16/20] Use global fetch if there is one --- packages/integrations/lit/server-shim.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/integrations/lit/server-shim.js b/packages/integrations/lit/server-shim.js index 9a4c7e408ed0..873d3cd8278e 100644 --- a/packages/integrations/lit/server-shim.js +++ b/packages/integrations/lit/server-shim.js @@ -1,5 +1,12 @@ import { installWindowOnGlobal } from '@lit-labs/ssr/lib/dom-shim.js'; -installWindowOnGlobal(); + +if(typeof fetch === 'function') { + const _fetch = fetch; + installWindowOnGlobal(); + globalThis.fetch = window.fetch = _fetch; +} else { + installWindowOnGlobal(); +} window.global = window; document.getElementsByTagName = () => []; From 371f91c78a298e75e272c3b2708bda5c8ace31a8 Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Mon, 9 Jan 2023 14:43:54 -0500 Subject: [PATCH 17/20] changeset for lit --- .changeset/chatty-rivers-camp.md | 5 +++++ packages/astro/test/test-utils.js | 9 +-------- 2 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 .changeset/chatty-rivers-camp.md diff --git a/.changeset/chatty-rivers-camp.md b/.changeset/chatty-rivers-camp.md new file mode 100644 index 000000000000..99e24e82e447 --- /dev/null +++ b/.changeset/chatty-rivers-camp.md @@ -0,0 +1,5 @@ +--- +'@astrojs/lit': patch +--- + +Only shim fetch if not already present diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index f0d705cb58c7..27e4caa5ea9c 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -157,10 +157,7 @@ export async function loadFixture(inlineConfig) { }, config, resolveUrl, - fetch: (url, init) => { - console.log('fetch2', fetch) - return fetch(resolveUrl(url), init) - }, + fetch: (url, init) => fetch(resolveUrl(url), init), preview: async (opts = {}) => { process.env.NODE_ENV = 'production'; const previewServer = await preview(settings, { logging, telemetry, ...opts }); @@ -308,10 +305,6 @@ export function fixLineEndings(str) { } export async function* streamAsyncIterator(stream) { - if(typeof stream.getReader !== 'function') { - console.log(stream); - throw new Error(`stream.getReader() is not a function.`); - } const reader = stream.getReader(); try { From 10a3e93724d5b04a1225dad19d5a4bcafb6b4705 Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Mon, 9 Jan 2023 16:00:23 -0500 Subject: [PATCH 18/20] Remove web-streams-polyfill --- packages/webapi/LICENSE | 2 -- packages/webapi/package.json | 3 +-- packages/webapi/run/build.js | 4 ++-- packages/webapi/src/ponyfill.ts | 2 +- packages/webapi/src/types.d.ts | 1 - pnpm-lock.yaml | 2 -- 6 files changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/webapi/LICENSE b/packages/webapi/LICENSE index 11ffac14c040..7dc74ec3874e 100644 --- a/packages/webapi/LICENSE +++ b/packages/webapi/LICENSE @@ -31,5 +31,3 @@ Code from [event-target-shim](https://www.npmjs.com/package/event-target-shim) i Code from [fetch-blob](https://www.npmjs.com/package/fetch-blob) is licensed under the MIT License (MIT), Copyright Jimmy Wärting. Code from [formdata-polyfill](https://www.npmjs.com/package/formdata-polyfill) is licensed under the MIT License (MIT), Copyright Jimmy Wärting. - -Code from [web-streams-polyfill](https://www.npmjs.com/package/web-streams-polyfill) is licensed under the MIT License (MIT), Copyright Mattias Buelens and Diwank Singh Tomer. diff --git a/packages/webapi/package.json b/packages/webapi/package.json index a64b31829ea4..e712e541d89c 100644 --- a/packages/webapi/package.json +++ b/packages/webapi/package.json @@ -74,8 +74,7 @@ "rollup-plugin-terser": "^7.0.2", "tslib": "^2.4.0", "typescript": "~4.7.3", - "urlpattern-polyfill": "^1.0.0-rc5", - "web-streams-polyfill": "^3.2.1" + "urlpattern-polyfill": "^1.0.0-rc5" }, "scripts": { "build": "node run/build.js", diff --git a/packages/webapi/run/build.js b/packages/webapi/run/build.js index cfb58f69745d..154d6ffbd57c 100644 --- a/packages/webapi/run/build.js +++ b/packages/webapi/run/build.js @@ -76,13 +76,13 @@ const plugins = [ MediaQueryList: ['./MediaQueryList', 'MediaQueryList'], Node: ['./Node', 'Node'], ReadableStream: [ - 'web-streams-polyfill/dist/ponyfill.es6.mjs', + 'node:stream/web', 'ReadableStream', ], ShadowRoot: ['./Node', 'ShadowRoot'], Window: ['./Window', 'Window'], 'globalThis.ReadableStream': [ - 'web-streams-polyfill/dist/ponyfill.es6.mjs', + 'node:stream/web', 'ReadableStream', ], }), diff --git a/packages/webapi/src/ponyfill.ts b/packages/webapi/src/ponyfill.ts index 671b036e8b20..a1088bdb4fb5 100644 --- a/packages/webapi/src/ponyfill.ts +++ b/packages/webapi/src/ponyfill.ts @@ -22,7 +22,7 @@ import { WritableStream, WritableStreamDefaultController, WritableStreamDefaultWriter, -} from 'web-streams-polyfill/dist/ponyfill.es6.mjs' +} from 'node:stream/web' import { cancelAnimationFrame, requestAnimationFrame, diff --git a/packages/webapi/src/types.d.ts b/packages/webapi/src/types.d.ts index 20db0d677c67..2597566eedd8 100644 --- a/packages/webapi/src/types.d.ts +++ b/packages/webapi/src/types.d.ts @@ -3,4 +3,3 @@ declare module '@ungap/structured-clone/esm/index.js' declare module '@ungap/structured-clone/esm/deserialize.js' declare module '@ungap/structured-clone/esm/serialize.js' declare module 'abort-controller/dist/abort-controller.mjs' -declare module 'web-streams-polyfill/dist/ponyfill.es6.mjs' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eaec9b658069..ba765f30835b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3579,7 +3579,6 @@ importers: typescript: ~4.7.3 undici: ^5.14.0 urlpattern-polyfill: ^1.0.0-rc5 - web-streams-polyfill: ^3.2.1 dependencies: global-agent: 3.0.0 undici: 5.14.0 @@ -3605,7 +3604,6 @@ importers: tslib: 2.4.1 typescript: 4.7.4 urlpattern-polyfill: 1.0.0-rc5 - web-streams-polyfill: 3.2.1 scripts: specifiers: From d0a67446f085412127b2df46667ff4dc3ab701f7 Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Mon, 9 Jan 2023 16:27:07 -0500 Subject: [PATCH 19/20] Remove web-streams-polyfill license note --- packages/webapi/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/webapi/README.md b/packages/webapi/README.md index adb31b26e3d3..2f7726e9a839 100644 --- a/packages/webapi/README.md +++ b/packages/webapi/README.md @@ -173,5 +173,3 @@ Code from [event-target-shim](https://www.npmjs.com/package/event-target-shim) i Code from [fetch-blob](https://www.npmjs.com/package/fetch-blob) is licensed under the MIT License (MIT), Copyright Jimmy Wärting. Code from [formdata-polyfill](https://www.npmjs.com/package/formdata-polyfill) is licensed under the MIT License (MIT), Copyright Jimmy Wärting. - -Code from [web-streams-polyfill](https://www.npmjs.com/package/web-streams-polyfill) is licensed under the MIT License (MIT), Copyright Mattias Buelens and Diwank Singh Tomer. From 53222176b5a18ade4acf8794b486fa47d1e43e43 Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Mon, 9 Jan 2023 16:45:21 -0500 Subject: [PATCH 20/20] Update .changeset/stupid-wolves-explain.md Co-authored-by: Nate Moore --- .changeset/stupid-wolves-explain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/stupid-wolves-explain.md b/.changeset/stupid-wolves-explain.md index f6597b087a09..742e90147ace 100644 --- a/.changeset/stupid-wolves-explain.md +++ b/.changeset/stupid-wolves-explain.md @@ -4,4 +4,4 @@ Replace node-fetch's polyfill with undici. -Since `undici` does not support it, this changes remove support for the `file:` protocol +Since `undici` does not support it, this change also removes custom support for the `file:` protocol