From 4848560caf6b142f85ec01db29ea75c70b25f1cd Mon Sep 17 00:00:00 2001 From: simov Date: Sat, 6 Jun 2015 10:47:37 +0300 Subject: [PATCH 1/3] Move copy method to helpers --- lib/copy.js | 10 ---------- lib/helpers.js | 9 +++++++++ request.js | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) delete mode 100644 lib/copy.js diff --git a/lib/copy.js b/lib/copy.js deleted file mode 100644 index ad162a508..000000000 --- a/lib/copy.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict' - -module.exports = -function copy (obj) { - var o = {} - Object.keys(obj).forEach(function (i) { - o[i] = obj[i] - }) - return o -} diff --git a/lib/helpers.js b/lib/helpers.js index 1d588ca94..5cc79da86 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -46,10 +46,19 @@ function toBase64 (str) { return (new Buffer(str || '', 'utf8')).toString('base64') } +function copy (obj) { + var o = {} + Object.keys(obj).forEach(function (i) { + o[i] = obj[i] + }) + return o +} + exports.isFunction = isFunction exports.paramsHaveRequestBody = paramsHaveRequestBody exports.safeStringify = safeStringify exports.md5 = md5 exports.isReadStream = isReadStream exports.toBase64 = toBase64 +exports.copy = copy exports.defer = deferMethod() diff --git a/request.js b/request.js index 21b4f5294..678c16741 100644 --- a/request.js +++ b/request.js @@ -6,7 +6,6 @@ var http = require('http') , util = require('util') , stream = require('stream') , zlib = require('zlib') - , helpers = require('./lib/helpers') , bl = require('bl') , hawk = require('hawk') , aws = require('aws-sign2') @@ -17,8 +16,8 @@ var http = require('http') , caseless = require('caseless') , ForeverAgent = require('forever-agent') , FormData = require('form-data') + , helpers = require('./lib/helpers') , cookies = require('./lib/cookies') - , copy = require('./lib/copy') , getProxyFromURI = require('./lib/getProxyFromURI') , Querystring = require('./lib/querystring').Querystring , Har = require('./lib/har').Har @@ -31,6 +30,7 @@ var safeStringify = helpers.safeStringify , isReadStream = helpers.isReadStream , toBase64 = helpers.toBase64 , defer = helpers.defer + , copy = helpers.copy , globalCookieJar = cookies.jar() From da7bab63d79a1ac8362346a3bce7bf76d165ea53 Mon Sep 17 00:00:00 2001 From: simov Date: Sat, 6 Jun 2015 12:55:53 +0300 Subject: [PATCH 2/3] Move tunnel logic into separate module --- lib/tunnel.js | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++ request.js | 171 ++----------------------------------------------- 2 files changed, 180 insertions(+), 164 deletions(-) create mode 100644 lib/tunnel.js diff --git a/lib/tunnel.js b/lib/tunnel.js new file mode 100644 index 000000000..bb0e2e4f7 --- /dev/null +++ b/lib/tunnel.js @@ -0,0 +1,173 @@ +'use strict' + +var url = require('url') + , tunnel = require('tunnel-agent') + +var defaultProxyHeaderWhiteList = [ + 'accept', + 'accept-charset', + 'accept-encoding', + 'accept-language', + 'accept-ranges', + 'cache-control', + 'content-encoding', + 'content-language', + 'content-length', + 'content-location', + 'content-md5', + 'content-range', + 'content-type', + 'connection', + 'date', + 'expect', + 'max-forwards', + 'pragma', + 'referer', + 'te', + 'transfer-encoding', + 'user-agent', + 'via' +] + +var defaultProxyHeaderExclusiveList = [ + 'proxy-authorization' +] + +function constructProxyHost(uriObject) { + var port = uriObject.portA + , protocol = uriObject.protocol + , proxyHost = uriObject.hostname + ':' + + if (port) { + proxyHost += port + } else if (protocol === 'https:') { + proxyHost += '443' + } else { + proxyHost += '80' + } + + return proxyHost +} + +function constructProxyHeaderWhiteList(headers, proxyHeaderWhiteList) { + var whiteList = proxyHeaderWhiteList + .reduce(function (set, header) { + set[header.toLowerCase()] = true + return set + }, {}) + + return Object.keys(headers) + .filter(function (header) { + return whiteList[header.toLowerCase()] + }) + .reduce(function (set, header) { + set[header] = headers[header] + return set + }, {}) +} + +function constructTunnelOptions (request) { + var proxy = request.proxy + + var tunnelOptions = { + proxy : { + host : proxy.hostname, + port : +proxy.port, + proxyAuth : proxy.auth, + headers : request.proxyHeaders + }, + headers : request.headers, + ca : request.ca, + cert : request.cert, + key : request.key, + passphrase : request.passphrase, + pfx : request.pfx, + ciphers : request.ciphers, + rejectUnauthorized : request.rejectUnauthorized, + secureOptions : request.secureOptions, + secureProtocol : request.secureProtocol + } + + return tunnelOptions +} + +function constructTunnelFnName(uri, proxy) { + var uriProtocol = (uri.protocol === 'https:' ? 'https' : 'http') + var proxyProtocol = (proxy.protocol === 'https:' ? 'Https' : 'Http') + return [uriProtocol, proxyProtocol].join('Over') +} + +function getTunnelFn(request) { + var uri = request.uri + var proxy = request.proxy + var tunnelFnName = constructTunnelFnName(uri, proxy) + return tunnel[tunnelFnName] +} + + +function Tunnel (request) { + this.request = request +} + +Tunnel.prototype.isEnabled = function (options) { + var request = this.request + // Tunnel HTTPS by default, or if a previous request in the redirect chain + // was tunneled. Allow the user to override this setting. + + // If self.tunnel is already set (because this is a redirect), use the + // existing value. + if (typeof request.tunnel !== 'undefined') { + return request.tunnel + } + + // If options.tunnel is set (the user specified a value), use it. + if (typeof options.tunnel !== 'undefined') { + return options.tunnel + } + + // If the destination is HTTPS, tunnel. + if (request.uri.protocol === 'https:') { + return true + } + + // Otherwise, leave tunnel unset, because if a later request in the redirect + // chain is HTTPS then that request (and any subsequent ones) should be + // tunneled. + return undefined +} + +Tunnel.prototype.setup = function () { + var self = this + , request = self.request + + if (typeof request.proxy === 'string') { + request.proxy = url.parse(request.proxy) + } + + if (!request.proxy || !request.tunnel) { + return false + } + + // Setup Proxy Header Exclusive List and White List + request.proxyHeaderExclusiveList = request.proxyHeaderExclusiveList || [] + request.proxyHeaderWhiteList = request.proxyHeaderWhiteList || defaultProxyHeaderWhiteList + var proxyHeaderExclusiveList = request.proxyHeaderExclusiveList.concat(defaultProxyHeaderExclusiveList) + var proxyHeaderWhiteList = request.proxyHeaderWhiteList.concat(proxyHeaderExclusiveList) + + // Setup Proxy Headers and Proxy Headers Host + // Only send the Proxy White Listed Header names + request.proxyHeaders = constructProxyHeaderWhiteList(request.headers, proxyHeaderWhiteList) + request.proxyHeaders.host = constructProxyHost(request.uri) + proxyHeaderExclusiveList.forEach(request.removeHeader, request) + + // Set Agent from Tunnel Data + var tunnelFn = getTunnelFn(request) + var tunnelOptions = constructTunnelOptions(request) + request.agent = tunnelFn(tunnelOptions) + + return true +} + +Tunnel.defaultProxyHeaderWhiteList = defaultProxyHeaderWhiteList +Tunnel.defaultProxyHeaderExclusiveList = defaultProxyHeaderExclusiveList +exports.Tunnel = Tunnel diff --git a/request.js b/request.js index 678c16741..c94d24089 100644 --- a/request.js +++ b/request.js @@ -11,7 +11,6 @@ var http = require('http') , aws = require('aws-sign2') , httpSignature = require('http-signature') , mime = require('mime-types') - , tunnel = require('tunnel-agent') , stringstream = require('stringstream') , caseless = require('caseless') , ForeverAgent = require('forever-agent') @@ -25,6 +24,7 @@ var http = require('http') , OAuth = require('./lib/oauth').OAuth , Multipart = require('./lib/multipart').Multipart , Redirect = require('./lib/redirect').Redirect + , Tunnel = require('./lib/tunnel').Tunnel var safeStringify = helpers.safeStringify , isReadStream = helpers.isReadStream @@ -36,36 +36,6 @@ var safeStringify = helpers.safeStringify var globalPool = {} -var defaultProxyHeaderWhiteList = [ - 'accept', - 'accept-charset', - 'accept-encoding', - 'accept-language', - 'accept-ranges', - 'cache-control', - 'content-encoding', - 'content-language', - 'content-length', - 'content-location', - 'content-md5', - 'content-range', - 'content-type', - 'connection', - 'date', - 'expect', - 'max-forwards', - 'pragma', - 'referer', - 'te', - 'transfer-encoding', - 'user-agent', - 'via' -] - -var defaultProxyHeaderExclusiveList = [ - 'proxy-authorization' -] - function filterForNonReserved(reserved, options) { // Filter out properties that are not reserved. // Reserved values are passed in at call site. @@ -96,103 +66,6 @@ function filterOutReservedFunctions(reserved, options) { } -function constructProxyHost(uriObject) { - var port = uriObject.portA - , protocol = uriObject.protocol - , proxyHost = uriObject.hostname + ':' - - if (port) { - proxyHost += port - } else if (protocol === 'https:') { - proxyHost += '443' - } else { - proxyHost += '80' - } - - return proxyHost -} - -function constructProxyHeaderWhiteList(headers, proxyHeaderWhiteList) { - var whiteList = proxyHeaderWhiteList - .reduce(function (set, header) { - set[header.toLowerCase()] = true - return set - }, {}) - - return Object.keys(headers) - .filter(function (header) { - return whiteList[header.toLowerCase()] - }) - .reduce(function (set, header) { - set[header] = headers[header] - return set - }, {}) -} - -function getTunnelOption(self, options) { - // Tunnel HTTPS by default, or if a previous request in the redirect chain - // was tunneled. Allow the user to override this setting. - - // If self.tunnel is already set (because this is a redirect), use the - // existing value. - if (typeof self.tunnel !== 'undefined') { - return self.tunnel - } - - // If options.tunnel is set (the user specified a value), use it. - if (typeof options.tunnel !== 'undefined') { - return options.tunnel - } - - // If the destination is HTTPS, tunnel. - if (self.uri.protocol === 'https:') { - return true - } - - // Otherwise, leave tunnel unset, because if a later request in the redirect - // chain is HTTPS then that request (and any subsequent ones) should be - // tunneled. - return undefined -} - -function constructTunnelOptions(request) { - var proxy = request.proxy - - var tunnelOptions = { - proxy : { - host : proxy.hostname, - port : +proxy.port, - proxyAuth : proxy.auth, - headers : request.proxyHeaders - }, - headers : request.headers, - ca : request.ca, - cert : request.cert, - key : request.key, - passphrase : request.passphrase, - pfx : request.pfx, - ciphers : request.ciphers, - rejectUnauthorized : request.rejectUnauthorized, - secureOptions : request.secureOptions, - secureProtocol : request.secureProtocol - } - - return tunnelOptions -} - -function constructTunnelFnName(uri, proxy) { - var uriProtocol = (uri.protocol === 'https:' ? 'https' : 'http') - var proxyProtocol = (proxy.protocol === 'https:' ? 'Https' : 'Http') - return [uriProtocol, proxyProtocol].join('Over') -} - -function getTunnelFn(request) { - var uri = request.uri - var proxy = request.proxy - var tunnelFnName = constructTunnelFnName(uri, proxy) - return tunnel[tunnelFnName] -} - // Function for properly handling a connection error function connectionErrorHandler(error) { var socket = this @@ -262,6 +135,7 @@ function Request (options) { self._oauth = new OAuth(self) self._multipart = new Multipart(self) self._redirect = new Redirect(self) + self._tunnel = new Tunnel(self) self.init(options) } @@ -276,37 +150,6 @@ function debug() { } Request.prototype.debug = debug -Request.prototype.setupTunnel = function () { - var self = this - - if (typeof self.proxy === 'string') { - self.proxy = url.parse(self.proxy) - } - - if (!self.proxy || !self.tunnel) { - return false - } - - // Setup Proxy Header Exclusive List and White List - self.proxyHeaderExclusiveList = self.proxyHeaderExclusiveList || [] - self.proxyHeaderWhiteList = self.proxyHeaderWhiteList || defaultProxyHeaderWhiteList - var proxyHeaderExclusiveList = self.proxyHeaderExclusiveList.concat(defaultProxyHeaderExclusiveList) - var proxyHeaderWhiteList = self.proxyHeaderWhiteList.concat(proxyHeaderExclusiveList) - - // Setup Proxy Headers and Proxy Headers Host - // Only send the Proxy White Listed Header names - self.proxyHeaders = constructProxyHeaderWhiteList(self.headers, proxyHeaderWhiteList) - self.proxyHeaders.host = constructProxyHost(self.uri) - proxyHeaderExclusiveList.forEach(self.removeHeader, self) - - // Set Agent from Tunnel Data - var tunnelFn = getTunnelFn(self) - var tunnelOptions = constructTunnelOptions(self) - self.agent = tunnelFn(tunnelOptions) - - return true -} - Request.prototype.init = function (options) { // init() contains all the code to setup the request object. // the actual outgoing request is not started until start() is called @@ -450,9 +293,9 @@ Request.prototype.init = function (options) { self.proxy = getProxyFromURI(self.uri) } - self.tunnel = getTunnelOption(self, options) + self.tunnel = self._tunnel.isEnabled(options) if (self.proxy) { - self.setupTunnel() + self._tunnel.setup() } self._redirect.onRequest(options) @@ -750,7 +593,7 @@ Request.prototype._updateProtocol = function () { // previously was doing http, now doing https // if it's https, then we might need to tunnel now. if (self.proxy) { - if (self.setupTunnel()) { + if (self._tunnel.setup()) { return } } @@ -1545,10 +1388,10 @@ Request.prototype.destroy = function () { } Request.defaultProxyHeaderWhiteList = - defaultProxyHeaderWhiteList.slice() + Tunnel.defaultProxyHeaderWhiteList.slice() Request.defaultProxyHeaderExclusiveList = - defaultProxyHeaderExclusiveList.slice() + Tunnel.defaultProxyHeaderExclusiveList.slice() // Exports From 6e50b3593be481cf53facb84fb2ff7a4e027c303 Mon Sep 17 00:00:00 2001 From: simov Date: Sat, 6 Jun 2015 15:52:22 +0300 Subject: [PATCH 3/3] Minor refactoring in the tunnel module: - proxyHeaderWhiteList, proxyHeaderExclusiveList and proxyHeaders are no longer public properties of the request instance --- lib/tunnel.js | 30 ++++++++++++++++++++---------- request.js | 2 +- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/tunnel.js b/lib/tunnel.js index bb0e2e4f7..cf28016e2 100644 --- a/lib/tunnel.js +++ b/lib/tunnel.js @@ -66,7 +66,7 @@ function constructProxyHeaderWhiteList(headers, proxyHeaderWhiteList) { }, {}) } -function constructTunnelOptions (request) { +function constructTunnelOptions (request, proxyHeaders) { var proxy = request.proxy var tunnelOptions = { @@ -74,7 +74,7 @@ function constructTunnelOptions (request) { host : proxy.hostname, port : +proxy.port, proxyAuth : proxy.auth, - headers : request.proxyHeaders + headers : proxyHeaders }, headers : request.headers, ca : request.ca, @@ -107,6 +107,8 @@ function getTunnelFn(request) { function Tunnel (request) { this.request = request + this.proxyHeaderWhiteList = defaultProxyHeaderWhiteList + this.proxyHeaderExclusiveList = [] } Tunnel.prototype.isEnabled = function (options) { @@ -136,10 +138,12 @@ Tunnel.prototype.isEnabled = function (options) { return undefined } -Tunnel.prototype.setup = function () { +Tunnel.prototype.setup = function (options) { var self = this , request = self.request + options = options || {} + if (typeof request.proxy === 'string') { request.proxy = url.parse(request.proxy) } @@ -149,20 +153,26 @@ Tunnel.prototype.setup = function () { } // Setup Proxy Header Exclusive List and White List - request.proxyHeaderExclusiveList = request.proxyHeaderExclusiveList || [] - request.proxyHeaderWhiteList = request.proxyHeaderWhiteList || defaultProxyHeaderWhiteList - var proxyHeaderExclusiveList = request.proxyHeaderExclusiveList.concat(defaultProxyHeaderExclusiveList) - var proxyHeaderWhiteList = request.proxyHeaderWhiteList.concat(proxyHeaderExclusiveList) + if (options.proxyHeaderWhiteList) { + self.proxyHeaderWhiteList = options.proxyHeaderWhiteList + } + if (options.proxyHeaderExclusiveList) { + self.proxyHeaderExclusiveList = options.proxyHeaderExclusiveList + } + + var proxyHeaderExclusiveList = self.proxyHeaderExclusiveList.concat(defaultProxyHeaderExclusiveList) + var proxyHeaderWhiteList = self.proxyHeaderWhiteList.concat(proxyHeaderExclusiveList) // Setup Proxy Headers and Proxy Headers Host // Only send the Proxy White Listed Header names - request.proxyHeaders = constructProxyHeaderWhiteList(request.headers, proxyHeaderWhiteList) - request.proxyHeaders.host = constructProxyHost(request.uri) + var proxyHeaders = constructProxyHeaderWhiteList(request.headers, proxyHeaderWhiteList) + proxyHeaders.host = constructProxyHost(request.uri) + proxyHeaderExclusiveList.forEach(request.removeHeader, request) // Set Agent from Tunnel Data var tunnelFn = getTunnelFn(request) - var tunnelOptions = constructTunnelOptions(request) + var tunnelOptions = constructTunnelOptions(request, proxyHeaders) request.agent = tunnelFn(tunnelOptions) return true diff --git a/request.js b/request.js index c94d24089..c032ea8f6 100644 --- a/request.js +++ b/request.js @@ -295,7 +295,7 @@ Request.prototype.init = function (options) { self.tunnel = self._tunnel.isEnabled(options) if (self.proxy) { - self._tunnel.setup() + self._tunnel.setup(options) } self._redirect.onRequest(options)