Implemented support for chunked transfer-encoding

lib/form_data.js

@@ -31,6 +31,8 @@ function FormData(options) {
   this._overheadLength = 0;
   this._valueLength = 0;
   this._valuesToMeasure = [];
+  this._chunked = false;
+  this._calculatedLength = null;
 
   CombinedStream.call(this);
 
@@ -79,7 +81,7 @@ FormData.prototype.append = function(field, value, options) {
 };
 
 FormData.prototype._trackLength = function(header, value, options) {
-  var valueLength = 0;
+  var valueLength = 0, hasLength = true;
 
   // used w/ getLengthSync(), when length is known.
   // e.g. for streaming directly from a remote server,
@@ -91,6 +93,8 @@ FormData.prototype._trackLength = function(header, value, options) {
     valueLength = value.length;
   } else if (typeof value === 'string') {
     valueLength = Buffer.byteLength(value);
+  } else {
+    hasLength = false;
   }
 
   this._valueLength += valueLength;
@@ -100,15 +104,24 @@ FormData.prototype._trackLength = function(header, value, options) {
     Buffer.byteLength(header) +
     FormData.LINE_BREAK.length;
 
-  // empty or either doesn't have path or not an http response
-  if (!value || ( !value.path && !(value.readable && value.hasOwnProperty('httpVersion')) )) {
+  if (!value) {
+    return;
+  }
+
+  // no need to bother with the length, we already know it
+  if (hasLength) {
     return;
   }
 
-  // no need to bother with the length
-  if (!options.knownLength) {
-    this._valuesToMeasure.push(value);
+  // empty or either doesn't have path or not an http response
+  if (( !value.path && !(value.readable && value.hasOwnProperty('httpVersion')) )) {
+    this._chunked = true;
+    this._calculatedLength = NaN;
+    return;
   }
+
+  // measure it later
+  this._valuesToMeasure.push(value);
 };
 
 FormData.prototype._lengthRetriever = function(value, callback) {
@@ -162,7 +175,8 @@ FormData.prototype._lengthRetriever = function(value, callback) {
 
   // something else
   } else {
-    callback('Unknown stream');
+    // As NaN, it will become chunked
+    callback(null, NaN);
   }
 };
 
@@ -296,6 +310,10 @@ FormData.prototype.getHeaders = function(userHeaders) {
     'content-type': 'multipart/form-data; boundary=' + this.getBoundary()
   };
 
+  if (this._chunked) {
+    formHeaders['transfer-encoding'] = 'chunked';
+  }
+
   for (header in userHeaders) {
     if (userHeaders.hasOwnProperty(header)) {
       formHeaders[header.toLowerCase()] = userHeaders[header];
@@ -353,14 +371,31 @@ FormData.prototype.getLengthSync = function() {
 FormData.prototype.hasKnownLength = function() {
   var hasKnownLength = true;
 
-  if (this._valuesToMeasure.length) {
+  if (this._valuesToMeasure.length || this._chunked) {
     hasKnownLength = false;
   }
 
   return hasKnownLength;
 };
 
+// Public API to check if the stream will be chunked
+FormData.prototype.isChunked = function() {
+  return this._chunked;
+};
+
 FormData.prototype.getLength = function(cb) {
+
+  // Have we already calculated the length? (or know that it's chunked)
+  if (this._calculatedLength !== null) {
+    if (this._chunked) {
+      // An error (or a NaN value) will be compatible with current libraries' behavior,
+      // i.e. request.js: https://github.com/request/request/blob/8162961dfdb73dc35a5a4bfeefb858c2ed2ccbb7/request.js#L572
+      return cb(new Error('This form-data does not have a length and should be sent with `Transfer-Encoding: chunked`'), NaN);
+    } else {
+      return cb(null, this._calculatedLength);
+    }
+  }
+
   var knownLength = this._overheadLength + this._valueLength;
 
   if (this._streams.length) {
@@ -372,7 +407,7 @@ FormData.prototype.getLength = function(cb) {
     return;
   }
 
-  asynckit.parallel(this._valuesToMeasure, this._lengthRetriever, function(err, values) {
+  asynckit.parallel(this._valuesToMeasure, this._lengthRetriever, (function(err, values) {
     if (err) {
       cb(err);
       return;
@@ -382,8 +417,15 @@ FormData.prototype.getLength = function(cb) {
       knownLength += length;
     });
 
+    this._chunked = isNaN(knownLength);
+    this._calculatedLength = knownLength;
+
+    if (this._chunked) {
+      return cb(new Error('This form-data does not have a length and should be sent with `Transfer-Encoding: chunked`'), knownLength);
+    }
+
     cb(null, knownLength);
-  });
+  }).bind(this));
 };
 
 FormData.prototype.submit = function(params, cb) {
@@ -426,15 +468,18 @@ FormData.prototype.submit = function(params, cb) {
 
   // get content length and fire away
   this.getLength(function(err, length) {
-    if (err) {
+    if (isNaN(length)) {
+      request.setHeader('Transfer-Encoding', 'chunked');
+    } else if (err) {
       this._error(err);
       return;
+    } else {
+      // add content length
+      request.setHeader('Content-Length', length);
     }
 
-    // add content length
-    request.setHeader('Content-Length', length);
-
     this.pipe(request);
+
     if (cb) {
       request.on('error', cb);
       request.on('response', cb.bind(this, null));

test/common.js

@@ -66,7 +66,7 @@ common.actions.populateFields = function(form, fields)
     if ((typeof field.value == 'function')) {
       field.value = field.value();
     }
-    form.append(name, field.value);
+    form.append(name, field.value, field.options);
   }
 };
 

test/integration/test-custom-headers-object.js

@@ -9,7 +9,12 @@ var http = require('http');
 
 var FormData = require(common.dir.lib + '/form_data');
 
-var testHeader = { 'X-Test-Fake': 123 };
+var testHeader = {
+  'X-Test-Fake': 123,
+
+  // Cover "skip nullish header" code branch
+  'X-Empty-Header': undefined
+};
 
 var expectedLength;
 
@@ -49,6 +54,13 @@ server.listen(common.port, function() {
 
   form.append('my_buffer', buffer, options);
 
+  // Cover the "userHeaders.hasOwnProperty(header)" code branch
+  var headers = form.getHeaders({
+    'x-custom-header': 'value'
+  });
+
+  assert.strictEqual(headers['X-Custom-Header'], 'value');
+
   // (available to req handler)
   expectedLength = form._lastBoundary().length + form._overheadLength + options.knownLength;
 

test/integration/test-form-get-length.js

@@ -3,6 +3,7 @@ var assert = common.assert;
 var FormData = require(common.dir.lib + '/form_data');
 var fake = require('fake').create();
 var fs = require('fs');
+var Stream = require('stream');
 
 (function testEmptyForm() {
   var form = new FormData();
@@ -89,3 +90,39 @@ var fs = require('fs');
   fake.expectAnytime(callback, [null, expectedLength]);
   form.getLength(callback);
 })();
+
+(function testPassthroughStreamData() {
+
+  var fields = [
+    {
+      name: 'my_field',
+      value: 'Test 123'
+    },
+    {
+      name: 'my_image',
+      value: fs.createReadStream(common.dir.fixture + '/unicycle.jpg').pipe(new Stream.PassThrough())
+    },
+    {
+      name: 'my_buffer',
+      value: new Buffer('123')
+    },
+    {
+      name: 'my_txt',
+      value: fs.createReadStream(common.dir.fixture + '/veggies.txt').pipe(new Stream.PassThrough())
+    }
+  ];
+
+  var form = new FormData();
+
+  fields.forEach(function(field) {
+    form.append(field.name, field.value);
+  });
+
+  var callback = fake.callback('testPassthroughStreamData-getLength');
+  fake.expectAnytime(callback, []);
+  form.getLength(function (err, length) {
+    assert.ok(err, 'getLength should send an error');
+    assert.ok(isNaN(length), 'length should be NaN');
+    callback();
+  });
+})();

test/integration/test-submit-chunked-request.js

@@ -0,0 +1,69 @@
+var common = require('../common');
+var assert = common.assert;
+var mime = require('mime-types');
+var request = require('request');
+var FormData = require(common.dir.lib + '/form_data');
+
+var remoteFile = 'http://localhost:' + common.staticPort + '/unicycle.jpg';
+
+// wrap non simple values into function
+// just to deal with ReadStream "autostart"
+var FIELDS = {
+  'remote_file': {
+    type: mime.lookup(common.dir.fixture + '/unicycle.jpg'),
+    value: function() {
+      return request(remoteFile)
+        .on('response', function(response) {
+          // Remove content-length header from response, force it to be chunked
+          delete response.headers['content-length'];
+        });
+    }
+  }
+};
+
+// count total
+var fieldsPassed = Object.keys(FIELDS).length;
+
+// prepare form-receiving http server
+var server = common.testFields(FIELDS, function(fields){
+  fieldsPassed = fields;
+});
+
+server.listen(common.port, function() {
+
+  var form = new FormData();
+
+  common.actions.populateFields(form, FIELDS);
+
+  // custom params object passed to submit
+  form.submit({
+    port: common.port,
+    path: '/'
+  }, function(err, res) {
+
+    if (err) {
+      throw err;
+    }
+
+    // Only now we know that it's a chunked stream, as form.submit() called getLength()
+
+    assert.strictEqual(form.getHeaders()['transfer-encoding'], 'chunked');
+    assert.strictEqual(form.isChunked(), true);
+
+    // Verify status code
+    assert.strictEqual(res.statusCode, 200);
+
+    // Try to get length again - that should take an already cached value and cover the _calculatedLength code branch
+    form.getLength(function (ex, length) {
+      assert.ok(isNaN(length));
+    });
+
+    res.resume();
+    server.close();
+  });
+
+});
+
+process.on('exit', function() {
+  assert.strictEqual(fieldsPassed, 0);
+});

test/integration/test-submit-chunked.js

@@ -0,0 +1,84 @@
+var common = require('../common');
+var assert = common.assert;
+var mime = require('mime-types');
+var fs = require('fs');
+var Stream = require('stream');
+var FormData = require(common.dir.lib + '/form_data');
+
+// wrap non simple values into function
+// just to deal with ReadStream "autostart"
+var FIELDS = {
+  'my_field': {
+    value: 'my_value'
+  },
+  'my_empty_field': {
+    value: '' // Will cover the (!value) code branch
+  },
+  'my_buffer': {
+    type: FormData.DEFAULT_CONTENT_TYPE,
+    value: common.defaultTypeValue
+  },
+  'my_file': {
+    type: mime.lookup(common.dir.fixture + '/unicycle.jpg'),
+    value: function() { return fs.createReadStream(common.dir.fixture + '/unicycle.jpg').pipe(new Stream.PassThrough()); },
+    options: {
+      filename: 'unicycle.jpg',
+    }
+  }
+};
+
+var TEST_FIELDS = {
+  'my_field': {
+    value: 'my_value'
+  },
+  'my_empty_field': {
+    value: ''
+  },
+  'my_buffer': {
+    type: FormData.DEFAULT_CONTENT_TYPE,
+    value: common.defaultTypeValue()
+  },
+  'my_file': {
+    type: mime.lookup(common.dir.fixture + '/unicycle.jpg'),
+    value: fs.createReadStream(common.dir.fixture + '/unicycle.jpg')
+  }
+};
+
+// count total
+var fieldsPassed = Object.keys(FIELDS).length;
+
+// prepare form-receiving http server
+var server = common.testFields(TEST_FIELDS, function(fields){
+  fieldsPassed = fields;
+});
+
+server.listen(common.port, function() {
+
+  var form = new FormData();
+
+  common.actions.populateFields(form, FIELDS);
+
+  assert.strictEqual(form.getHeaders()['transfer-encoding'], 'chunked');
+  assert.strictEqual(form.isChunked(), true);
+
+  // custom params object passed to submit
+  form.submit({
+    port: common.port,
+    path: '/'
+  }, function(err, res) {
+
+    if (err) {
+      throw err;
+    }
+
+    assert.strictEqual(res.statusCode, 200);
+
+    res.resume();
+    server.close();
+  });
+
+});
+
+process.on('exit', function() {
+  assert.strictEqual(fieldsPassed, 0);
+});