Use websockets to stub large XHR response bodies instead of hea… (#5525)

* server: add test for XHR with body > 100kb via CLI

* Use websockets to stub large XHR response bodies instead of headers

* Properly cleanup outstanding XHRs on before:test:run

* Add lil unit test for xhr_ws_server

* Use reset:xhr:server to get around sending entire test:before:run:async payload

* Responding to feedback

* Implement feedback

* move data obj wrapping into xhrs controller

packages/driver/src/cy/commands/xhr.coffee

@@ -87,6 +87,9 @@ startXhrServer = (cy, state, config) ->
     xhrUrl: config("xhrUrl")
     stripOrigin: stripOrigin
 
+    emitIncoming: (id, route) ->
+      Cypress.backend('incoming:xhr', id, route)
+
     ## shouldnt these stubs be called routes?
     ## rename everything related to stubs => routes
     onSend: (xhr, stack, route) =>
@@ -246,6 +249,10 @@ module.exports = (Commands, Cypress, cy, state, config) ->
   ## correctly
   Cypress.on("window:unload", cancelPendingXhrs)
 
+  Cypress.on "test:before:run:async", ->
+    ## reset any state on the backend
+    Cypress.backend('reset:xhr:server')
+
   Cypress.on "test:before:run", ->
     ## reset the existing server
     reset()
@@ -256,7 +263,7 @@ module.exports = (Commands, Cypress, cy, state, config) ->
     ## window such as if the last test ended
     ## with a cross origin window
     try
-      server = startXhrServer(cy, state, config)
+      server = startXhrServer(cy, state, config, Cypress)
     catch err
       ## in this case, just don't bind to the server
       server = null

packages/driver/src/cypress/server.coffee

@@ -10,11 +10,8 @@ props                    = "onreadystatechange onload onerror".split(" ")
 
 restoreFn = null
 
-setHeader = (xhr, key, val, transformer) ->
+setHeader = (xhr, key, val) ->
   if val?
-    if transformer
-      val = transformer(val)
-
     key = "X-Cypress-" + _.capitalize(key)
     xhr.setRequestHeader(key, encodeURI(val))
 
@@ -176,18 +173,30 @@ create = (options = {}) ->
       hasEnabledStubs and route and route.response?
 
     applyStubProperties: (xhr, route) ->
-      responser = if _.isObject(route.response) then JSON.stringify else null
-
-      ## add header properties for the xhr's id
-      ## and the testId
-      setHeader(xhr, "id", xhr.id)
-      # setHeader(xhr, "testId", options.testId)
-
-      setHeader(xhr, "status",   route.status)
-      setHeader(xhr, "response", route.response, responser)
-      setHeader(xhr, "matched",  route.url + "")
-      setHeader(xhr, "delay",    route.delay)
-      setHeader(xhr, "headers",  route.headers, transformHeaders)
+      responseToString = =>
+        if not _.isString(route.response)
+          return JSON.stringify(route.response)
+
+        route.response
+
+      response = responseToString()
+
+      headers = {
+        "id": xhr.id
+        "status": route.status
+        "matched": route.url + ""
+        "delay": route.delay
+        "headers": transformHeaders(route.headers)
+      }
+
+      if response.length > 4096
+        options.emitIncoming(xhr.id, response)
+        headers.responseDeferred = true
+      else
+        headers.response = response
+
+      _.map headers, (v, k) =>
+        setHeader(xhr, k, v)
 
     route: (attrs = {}) ->
       warnOnStubDeprecation(attrs, "route")

packages/server/__snapshots__/4_xhr_spec.coffee.js

@@ -23,20 +23,21 @@ exports['e2e xhr / passes'] = `
     ✓ does not inject into json's contents from http server even requesting text/html
     ✓ does not inject into json's contents from file server even requesting text/html
     ✓ works prior to visit
+    ✓ can stub a 100kb response
     server with 1 visit
       ✓ response body
       ✓ request body
       ✓ aborts
 
 
-  8 passing
+  9 passing
 
 
   (Results)
 
   ┌────────────────────────────────────────────────────────────────────────────────────────────────┐
-  │ Tests:        8                                                                                │
-  │ Passing:      8                                                                                │
+  │ Tests:        9                                                                                │
+  │ Passing:      9                                                                                │
   │ Failing:      0                                                                                │
   │ Pending:      0                                                                                │
   │ Skipped:      0                                                                                │
@@ -60,9 +61,9 @@ exports['e2e xhr / passes'] = `
 
        Spec                                              Tests  Passing  Failing  Pending  Skipped  
   ┌────────────────────────────────────────────────────────────────────────────────────────────────┐
-  │ ✔  xhr_spec.coffee                          XX:XX        8        8        -        -        - │
+  │ ✔  xhr_spec.coffee                          XX:XX        9        9        -        -        - │
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
-    ✔  All specs passed!                        XX:XX        8        8        -        -        -  
+    ✔  All specs passed!                        XX:XX        9        9        -        -        -  
 
 
 `

packages/server/lib/controllers/xhrs.coffee

@@ -18,7 +18,7 @@ isValidJSON = (text) ->
   return false
 
 module.exports = {
-  handle: (req, res, config, next) ->
+  handle: (req, res, getDeferredResponse, config, next) ->
     get = (val, def) ->
       decodeURI(req.get(val) ? def)
 
@@ -27,6 +27,10 @@ module.exports = {
     headers  = get("x-cypress-headers", null)
     response = get("x-cypress-response", "")
 
+    if get("x-cypress-responsedeferred", "")
+      id = get("x-cypress-id")
+      response = getDeferredResponse(id)
+
     respond = =>
       ## figure out the stream interface and pipe these
       ## chunks to the response
@@ -54,6 +58,10 @@ module.exports = {
         .set(headers)
         .status(status)
         .end(chunk)
+      .catch { testEndedBeforeResponseReceived: true }, ->
+        res
+        .socket
+        .destroy()
       .catch (err) ->
         res
         .status(400)
@@ -79,10 +87,15 @@ module.exports = {
       {data: bytes, encoding: encoding}
 
   getResponse: (resp, config) ->
+    if resp.then
+      return resp
+      .then (data) =>
+        { data }
+
     if fixturesRe.test(resp)
-      @_get(resp, config)
-    else
-      Promise.resolve({data: resp})
+      return @_get(resp, config)
+
+    Promise.resolve({data: resp})
 
   parseContentType: (response) ->
     ret = (type) ->

packages/server/lib/routes.coffee

@@ -15,7 +15,7 @@ files       = require("./controllers/files")
 proxy       = require("./controllers/proxy")
 staticCtrl  = require("./controllers/static")
 
-module.exports = (app, config, request, getRemoteState, project, nodeProxy) ->
+module.exports = (app, config, request, getRemoteState, getDeferredResponse, project, nodeProxy) ->
   ## routing for the actual specs which are processed automatically
   ## this could be just a regular .js file or a .coffee file
   app.get "/__cypress/tests", (req, res, next) ->
@@ -45,7 +45,7 @@ module.exports = (app, config, request, getRemoteState, project, nodeProxy) ->
     files.handleIframe(req, res, config, getRemoteState)
 
   app.all "/__cypress/xhrs/*", (req, res, next) ->
-    xhrs.handle(req, res, config, next)
+    xhrs.handle(req, res, getDeferredResponse, config, next)
 
   app.get "/__root/*", (req, res, next) ->
     file = path.join(config.projectRoot, req.params[0])

packages/server/lib/server.coffee

@@ -30,6 +30,7 @@ logger       = require("./logger")
 Socket       = require("./socket")
 Request      = require("./request")
 fileServer   = require("./file_server")
+XhrServer    = require("./xhr_ws_server")
 templateEngine = require("./template_engine")
 
 DEFAULT_DOMAIN_NAME    = "localhost"
@@ -141,12 +142,13 @@ class Server
       ## and set the responseTimeout
       @_request = Request({timeout: config.responseTimeout})
       @_nodeProxy = httpProxy.createProxyServer()
+      @_xhrServer = XhrServer.create()
 
       getRemoteState = => @_getRemoteState()
 
       @createHosts(config.hosts)
 
-      @createRoutes(app, config, @_request, getRemoteState, project, @_nodeProxy)
+      @createRoutes(app, config, @_request, getRemoteState, @_xhrServer.getDeferredResponse, project, @_nodeProxy)
 
       @createServer(app, config, project, @_request, onWarning)
 
@@ -698,10 +700,11 @@ class Server
   startWebsockets: (automation, config, options = {}) ->
     options.onResolveUrl = @_onResolveUrl.bind(@)
     options.onRequest    = @_onRequest.bind(@)
+    options.onIncomingXhr = @_xhrServer.onIncomingXhr
+    options.onResetXhrServer = @_xhrServer.reset
 
     @_socket = Socket(config)
     @_socket.startListening(@_server, automation, config, options)
     @_normalizeReqUrl(@_server)
-    # handleListeners(@_server)
 
 module.exports = Server

packages/server/lib/socket.coffee

@@ -132,6 +132,8 @@ class Socket
 
     _.defaults options,
       socketId: null
+      onIncomingXhr: ->
+      onResetXhrServer: ->
       onSetRunnables: ->
       onMocha: ->
       onConnect: ->
@@ -298,6 +300,10 @@ class Socket
               options.onResolveUrl(url, headers, automationRequest, resolveOpts)
             when "http:request"
               options.onRequest(headers, automationRequest, args[0])
+            when "reset:xhr:server"
+              options.onResetXhrServer()
+            when "incoming:xhr"
+              options.onIncomingXhr(args[0], args[1])
             when "get:fixture"
               fixture.get(config.fixturesFolder, args[0], args[1])
             when "read:file"
@@ -343,6 +349,7 @@ class Socket
         socket.on event, (data) =>
           @toReporter(event, data)
 
+
   end: ->
     @ended = true
 

packages/server/lib/xhr_ws_server.ts

@@ -0,0 +1,93 @@
+import _ from 'lodash'
+import Bluebird from 'bluebird'
+import debugModule from 'debug'
+
+const debug = debugModule('cypress:server:xhr_ws_server')
+
+function trunc (str) {
+  return _.truncate(str, {
+    length: 100,
+    omission: '... [truncated to 100 chars]',
+  })
+}
+
+type DeferredPromise<T> = {
+  resolve: Function
+  reject: Function
+  promise: Bluebird<T>
+}
+
+export function create () {
+  let incomingXhrResponses: {
+    [key: string]: string | DeferredPromise<string>
+  } = {}
+
+  function onIncomingXhr (id: string, data: string) {
+    debug('onIncomingXhr %o', { id, res: trunc(data) })
+    const deferred = incomingXhrResponses[id]
+
+    if (deferred && typeof deferred !== 'string') {
+      // request came before response, resolve with it
+      return deferred.resolve(data)
+    }
+
+    // response came before request, cache the data
+    incomingXhrResponses[id] = data
+  }
+
+  function getDeferredResponse (id) {
+    debug('getDeferredResponse %o', { id })
+    // if we already have it, send it
+    const res = incomingXhrResponses[id]
+
+    if (res) {
+      if (typeof res === 'object') {
+        debug('returning existing deferred promise for %o', { id, res })
+
+        return res.promise
+      }
+
+      debug('already have deferred response %o', { id, res: trunc(res) })
+      delete incomingXhrResponses[id]
+
+      return res
+    }
+
+    let deferred: Partial<DeferredPromise<string>> = {}
+
+    deferred.promise = new Bluebird((resolve, reject) => {
+      debug('do not have response, waiting %o', { id })
+      deferred.resolve = resolve
+      deferred.reject = reject
+    })
+    .tap((res) => {
+      debug('deferred response found %o', { id, res: trunc(res) })
+    }) as Bluebird<string>
+
+    incomingXhrResponses[id] = deferred as DeferredPromise<string>
+
+    return deferred.promise
+  }
+
+  function reset () {
+    debug('resetting incomingXhrs %o', { incomingXhrResponses })
+
+    _.forEach(incomingXhrResponses, (res) => {
+      if (typeof res !== 'string') {
+        const err: any = new Error('This stubbed XHR was pending on a stub response object from the driver, but the test ended before that happened.')
+
+        err.testEndedBeforeResponseReceived = true
+
+        res.reject(err)
+      }
+    })
+
+    incomingXhrResponses = {}
+  }
+
+  return {
+    onIncomingXhr,
+    getDeferredResponse,
+    reset,
+  }
+}

packages/server/test/integration/http_requests_spec.coffee

@@ -667,6 +667,23 @@ describe "Routes", ->
           expect(res.statusCode).to.eq(200)
           expect(res.body).to.deep.eq({test: "We’ll"})
 
+      context "deferred", ->
+        it "closes connection if no stub is received before a reset", ->
+          p = @rp({
+            url: "http://localhost:2020/__cypress/xhrs/users/1"
+            json: true
+            headers: {
+              "x-cypress-id": "foo1"
+              "x-cypress-responsedeferred": true
+            }
+          })
+
+          setTimeout =>
+            @server._xhrServer.reset()
+          , 100
+
+          expect(p).to.be.rejectedWith('Error: socket hang up')
+
       context "fixture", ->
         beforeEach ->
           Fixtures.scaffold("todos")