fix: tests in CI (#1034)

Node.js v16.14.0 included [a fix](nodejs/node#40706) that meant that `@web-std/blob` started using the Node.js native version. Bad news, Node.js currently copies the buffer on every iteration when obtaining a stream from `File.stream()`. It also has a fixed and small chunk size of `65536` bytes. This makes reading the stream VERY slow and this test fails because it times out. I opened an issue about this here: nodejs/node#42108 Once the test was fixed, the cloudflare build for the website started failing so I had to update next.js to v12. WTF!?! After that, the client build started failing with: ``` Error: Build failed with 2 errors: ../../node_modules/parse-link-header/index.js:3:17: error: Could not resolve "querystring" (use "platform: 'node'" when building for node) ../../node_modules/parse-link-header/index.js:4:18: error: Could not resolve "url" (use "platform: 'node'" when building for node) at failureErrorWithLog (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:1493:15) at /Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:1151:28 at runOnEndCallbacks (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:941:63) at buildResponseToResult (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:1149:7) at /Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:1258:14 at /Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:629:9 at handleIncomingPacket (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:726:9) at Socket.readFromStdout (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:596:7) at Socket.emit (node:events:520:28) at addChunk (node:internal/streams/readable:315:12) { errors: [ { detail: undefined, location: [Object], notes: [], pluginName: '', text: `Could not resolve "querystring" (use "platform: 'node'" when building for node)` }, { detail: undefined, location: [Object], notes: [], pluginName: '', text: `Could not resolve "url" (use "platform: 'node'" when building for node)` } ], warnings: [] } ``` So I had to roll the update to `parse-link-header` from #1032 into here as well.
web3-storage · Feb 24, 2022 · ad26c27 · ad26c27
1 parent 0e5d0f8
commit ad26c27
Show file tree

Hide file tree

Showing 7 changed files with 13,473 additions and 8,042 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/client/package.json b/packages/client/package.json
@@ -31,18 +31,18 @@
   },
   "dependencies": {
     "@ipld/car": "^3.1.4",
-    "@web-std/blob": "^3.0.1",
-    "@web-std/fetch": "^3.0.0",
-    "@web-std/file": "^3.0.0",
+    "@web-std/blob": "^3.0.3",
+    "@web-std/fetch": "^3.0.3",
+    "@web-std/file": "^3.0.2",
+    "@web3-storage/parse-link-header": "^3.1.0",
     "browser-readablestream-to-it": "^1.0.3",
     "carbites": "^1.0.6",
     "cborg": "^1.6.0",
     "files-from-path": "^0.2.3",
-    "ipfs-car": "^0.6.1",
+    "ipfs-car": "^0.6.2",
     "ipns": "^0.16.0",
     "libp2p-crypto": "^0.21.0",
     "p-retry": "^4.5.0",
-    "parse-link-header": "^2.0.0",
     "streaming-iterables": "^6.0.0",
     "uint8arrays": "^3.0.0"
   },
@@ -52,7 +52,6 @@
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^13.1.3",
     "@types/mocha": "9.0.0",
-    "@types/parse-link-header": "^1.0.1",
     "bundlesize": "^0.18.1",
     "cors": "^2.8.5",
     "del-cli": "^4.0.0",
@@ -62,6 +61,7 @@
     "npm-run-all": "^4.1.5",
     "nyc": "15.1.0",
     "playwright-test": "^7.2.2",
+    "randombytes": "^2.1.0",
     "rollup": "2.63.0",
     "rollup-plugin-multi-input": "1.3.1",
     "rollup-plugin-terser": "^7.0.2",

diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js
@@ -16,7 +16,7 @@
 import { transform } from 'streaming-iterables'
 import pRetry from 'p-retry'
 import { pack } from 'ipfs-car/pack'
-import parseLink from 'parse-link-header'
+import { parseLinkHeader } from '@web3-storage/parse-link-header'
 import { unpackStream } from 'ipfs-car/unpack'
 import { TreewalkCarSplitter } from 'carbites/treewalk'
 import { CarReader } from '@ipld/car'
@@ -469,13 +469,13 @@ function toWeb3Response (res) {
 async function * paginator (fn, service, opts) {
   let res = await fn(service, opts)
   yield res
-  let link = parseLink(res.headers.get('Link') || '')
+  let link = parseLinkHeader(res.headers.get('Link') || '')
   // @ts-ignore
   while (link && link.next) {
     // @ts-ignore
     res = await fn(service, link.next)
     yield res
-    link = parseLink(res.headers.get('Link') || '')
+    link = parseLinkHeader(res.headers.get('Link') || '')
   }
 }
 

diff --git a/packages/client/test/put.spec.js b/packages/client/test/put.spec.js
@@ -9,6 +9,7 @@ import { CID } from 'multiformats/cid'
 import { encode } from 'multiformats/block'
 import * as json from '@ipld/dag-json'
 import { sha256 } from 'multiformats/hashes/sha2'
+import { ReadableStream } from '@web-std/blob'
 
 describe('put', () => {
   const { AUTH_TOKEN, API_PORT } = process.env
@@ -85,12 +86,31 @@ describe('put', () => {
   })
 
   it('adds big files', async function () {
-    this.timeout(30e3)
+    this.timeout(60e3)
     const client = new Web3Storage({ token, endpoint })
     let uploadedChunks = 0
 
     const files = [
-      new File([randomBytes(1024e6)], '102mb.txt')
+      // Previously: new File([randomBytes(1024e6)], '102mb.txt')
+      //
+      // Node.js currently copies the buffer on every iteration when obtaining a
+      // stream from File.stream(). It also has a fixed and small chunk size of
+      // 65536 bytes. This makes reading the stream VERY slow and this test
+      // fails because it times out.
+      //
+      // TODO: revert to using File if this issue gets resolved:
+      // https://github.com/nodejs/node/issues/42108
+      {
+        name: '102mb.txt',
+        stream () {
+          return new ReadableStream({
+            pull (controller) {
+              controller.enqueue(randomBytes(1024e6))
+              controller.close()
+            }
+          })
+        }
+      }
     ]
 
     await client.put(files, {

diff --git a/packages/website/next-env.d.ts b/packages/website/next-env.d.ts
@@ -1,5 +1,4 @@
 /// <reference types="next" />
-/// <reference types="next/types/global" />
 /// <reference types="next/image-types/global" />
 
 // NOTE: This file should not be edited

diff --git a/packages/website/package.json b/packages/website/package.json
@@ -16,7 +16,7 @@
     "filesize": "^6.1.0",
     "gray-matter": "^4.0.3",
     "magic-sdk": "^4.2.1",
-    "next": "^11.0.1",
+    "next": "^12.1.0",
     "p-map": "^5.1.0",
     "pretty-bytes": "^5.6.0",
     "rc-tooltip": "^5.1.1",

diff --git a/packages/website/tsconfig.json b/packages/website/tsconfig.json
@@ -4,7 +4,10 @@
     "allowJs": true,
     "checkJs": true,
     "target": "esnext",
-    "lib": ["ESNext", "DOM"],
+    "lib": [
+      "ESNext",
+      "DOM"
+    ],
     "strict": true,
     "moduleResolution": "node",
     "sourceMap": true,
@@ -17,11 +20,21 @@
     "resolveJsonModule": true,
     "isolatedModules": true,
     "paths": {
-      "web3.storage": ["../client"]
-    }
+      "web3.storage": [
+        "../client"
+      ]
+    },
+    "incremental": true
   },
-  "include": ["lib", "pages", "components"],
-  "exclude": ["node_modules/", ".next/"],
+  "include": [
+    "lib",
+    "pages",
+    "components"
+  ],
+  "exclude": [
+    "node_modules/",
+    ".next/"
+  ],
   "references": [
     {
       "path": "../client"