From 71081b64ac939b74eaf7a510fee45ab27d4669a1 Mon Sep 17 00:00:00 2001
From: Anna Henningsen <anna@addaleax.net>
Date: Wed, 21 Dec 2016 07:21:38 +0100
Subject: [PATCH] More lenient gzip decompression
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Be explicitly lenient with gzip decompression by always requesting
`zlib` to flush the input data and never explicitly ending the
`zlib` input.

The behavioural difference is that on Node ≥ 6, which has a slightly
stricter gzip decoding process than previous Node versions, malformed
but otherwise acceptable server responses are still properly
decompressed (the most common example being a missing checksum
at the stream end).

This aligns behaviour with cURL, which always uses the `Z_SYNC_FLUSH`
flag for decompression.

On the downside, accidental truncation of a response is no longer
detected on the compression layer.

Ref: https://github.com/nodejs/node/issues/8701#issuecomment-268224481
Fixes: https://github.com/request/request/issues/2482
---
 request.js         | 13 +++++++++++--
 tests/test-gzip.js | 16 ++++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/request.js b/request.js
index 26f70c208..d9ade74c9 100644
--- a/request.js
+++ b/request.js
@@ -946,11 +946,20 @@ Request.prototype.onRequestResponse = function (response) {
       var contentEncoding = response.headers['content-encoding'] || 'identity'
       contentEncoding = contentEncoding.trim().toLowerCase()
 
+      // Be more lenient with decoding compressed responses, since (very rarely)
+      // servers send slightly invalid gzip responses that are still accepted
+      // by common browsers.
+      // Always using Z_SYNC_FLUSH is what cURL does.
+      var zlibOptions = {
+        flush: zlib.Z_SYNC_FLUSH
+      , finishFlush: zlib.Z_SYNC_FLUSH
+      }
+
       if (contentEncoding === 'gzip') {
-        responseContent = zlib.createGunzip()
+        responseContent = zlib.createGunzip(zlibOptions)
         response.pipe(responseContent)
       } else if (contentEncoding === 'deflate') {
-        responseContent = zlib.createInflate()
+        responseContent = zlib.createInflate(zlibOptions)
         response.pipe(responseContent)
       } else {
         // Since previous versions didn't check for Content-Encoding header,
diff --git a/tests/test-gzip.js b/tests/test-gzip.js
index 7ade4aee8..ac523a8d7 100644
--- a/tests/test-gzip.js
+++ b/tests/test-gzip.js
@@ -39,6 +39,12 @@ var server = http.createServer(function(req, res) {
       res.writeHead(200)
       res.write(testContentBigGzip.slice(0, 4096))
       setTimeout(function() { res.end(testContentBigGzip.slice(4096)) }, 10)
+    } else if (req.url === '/just-slightly-truncated') {
+      zlib.gzip(testContent, function(err, data) {
+        assert.equal(err, null)
+        // truncate the CRC checksum and size check at the end of the stream
+        res.end(data.slice(0, data.length-8))
+      })
     } else {
       zlib.gzip(testContent, function(err, data) {
         assert.equal(err, null)
@@ -96,6 +102,16 @@ tape('transparently supports gzip decoding to callbacks', function(t) {
   })
 })
 
+tape('supports slightly invalid gzip content', function(t) {
+  var options = { url: server.url + '/just-slightly-truncated', gzip: true }
+  request.get(options, function(err, res, body) {
+    t.equal(err, null)
+    t.equal(res.headers['content-encoding'], 'gzip')
+    t.equal(body, testContent)
+    t.end()
+  })
+})
+
 tape('transparently supports gzip decoding to pipes', function(t) {
   var options = { url: server.url + '/foo', gzip: true }
   var chunks = []