Rewrite providers to use streams

This removes the need for disk space as data will be buffered in memory and backpressure will be respected
#3098 (comment)
All providers "download" methods will now return a { stream } which can be consumed by uploader.

Also:
- Remove capacitor (no longer needed)
- Change Provider/SearchProvider API to async (Breaking change for custom companion providers)
- Fix the case with unknown length streams (zoom / google drive). Need to be downloaded first
- rewrite controllers deauth-callback, thumbnail, list, logout to async
- getURLMeta: make sure size is never NaN (NaN gets converted to null in JSON.stringify when sent to client but not when used in backend)
- fix purest mock (it wasn't returning statusCode on('response'))
- add missing http mock for "request" for THUMBNAIL_URL and http://url.myendpoint.com/file (these request errors were never caught by tests previously)
- "upload functions with tus protocol" test: move filename checking to new test where size is null. Fix broken expects
- fix some lint

examples/custom-provider/server/customprovider.js

@@ -36,7 +36,8 @@ class MyCustomProvider {
     this.authProvider = 'myunsplash'
   }
 
-  list ({ token, directory }, done) {
+  // eslint-disable-next-line class-methods-use-this
+  async list ({ token, directory }) {
     const path = directory ? `/${directory}/photos` : ''
     const options = {
       url: `${BASE_URL}/collections${path}`,
@@ -47,18 +48,20 @@ class MyCustomProvider {
       },
     }
 
-    request(options, (err, resp, body) => {
-      if (err) {
-        console.log(err)
-        done(err)
-        return
-      }
+    return new Promise((resolve, reject) => (
+      request(options, (err, resp, body) => {
+        if (err) {
+          console.log(err)
+          reject(err)
+          return
+        }
 
-      done(null, adaptData(body))
-    })
+        resolve(adaptData(body))
+      })))
   }
 
-  download ({ id, token }, onData) {
+  // eslint-disable-next-line class-methods-use-this
+  async download ({ id, token }) {
     const options = {
       url: `${BASE_URL}/photos/${id}`,
       method: 'GET',
@@ -68,21 +71,27 @@ class MyCustomProvider {
       },
     }
 
-    request(options, (err, resp, body) => {
-      if (err) {
-        console.log(err)
-        return
-      }
-
-      const url = body.links.download
-      request.get(url)
-        .on('data', (chunk) => onData(null, chunk))
-        .on('end', () => onData(null, null))
-        .on('error', (err) => console.log(err))
-    })
+    const resp = await new Promise((resolve, reject) => (
+      request(options)
+        .on('response', (response) => {
+          // Don't allow any more data to flow yet.
+          // https://github.com/request/request/issues/1990#issuecomment-184712275
+          response.pause()
+          resolve(response)
+        })
+        .on('error', reject)
+    ))
+
+    if (resp.statusCode !== 200) {
+      throw new Error(`HTTP response ${resp.statusCode}`)
+    }
+
+    // The returned stream will be consumed and uploaded from the current position
+    return { stream: resp }
   }
 
-  size ({ id, token }, done) {
+  // eslint-disable-next-line class-methods-use-this
+  async size ({ id, token }) {
     const options = {
       url: `${BASE_URL}/photos/${id}`,
       method: 'GET',
@@ -92,15 +101,16 @@ class MyCustomProvider {
       },
     }
 
-    request(options, (err, resp, body) => {
-      if (err) {
-        console.log(err)
-        done(err)
-        return
-      }
+    return new Promise((resolve, reject) => (
+      request(options, (err, resp, body) => {
+        if (err) {
+          console.log(err)
+          reject(err)
+          return
+        }
 
-      done(null, body.width * body.height)
-    })
+        resolve(body.size)
+      })))
   }
 }
 

packages/@uppy/companion/package.json

@@ -45,7 +45,6 @@
     "express-prom-bundle": "6.3.0",
     "express-request-id": "1.4.1",
     "express-session": "1.17.1",
-    "fs-capacitor": "^7.0.1",
     "grant": "4.7.0",
     "helmet": "^4.6.0",
     "ip-address": "6.2.0",
@@ -88,6 +87,8 @@
     "@types/uuid": "3.4.7",
     "@types/webpack": "^5.28.0",
     "@types/ws": "6.0.4",
+    "into-stream": "^6.0.0",
+    "nock": "^13.1.3",
     "supertest": "3.4.2",
     "typescript": "~4.3"
   },