/
request.js
155 lines (132 loc) · 4.86 KB
/
request.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
// eslint-disable-next-line max-classes-per-file
const http = require('node:http')
const https = require('node:https')
const { URL } = require('node:url')
const dns = require('node:dns')
const request = require('request')
const ipaddr = require('ipaddr.js')
const logger = require('../logger')
const FORBIDDEN_IP_ADDRESS = 'Forbidden IP address'
// Example scary IPs that should return false (ipv6-to-ipv4 mapped):
// ::FFFF:127.0.0.1
// ::ffff:7f00:1
const isDisallowedIP = (ipAddress) => ipaddr.parse(ipAddress).range() !== 'unicast'
module.exports.FORBIDDEN_IP_ADDRESS = FORBIDDEN_IP_ADDRESS
module.exports.getRedirectEvaluator = (rawRequestURL, blockPrivateIPs) => {
const requestURL = new URL(rawRequestURL)
return (res) => {
if (!blockPrivateIPs) {
return true
}
let redirectURL = null
try {
redirectURL = new URL(res.headers.location, requestURL)
} catch (err) {
return false
}
const shouldRedirect = redirectURL.protocol === requestURL.protocol
if (!shouldRedirect) {
logger.info(
`blocking redirect from ${requestURL} to ${redirectURL}`, 'redirect.protection',
)
}
return shouldRedirect
}
}
function dnsLookup (hostname, options, callback) {
dns.lookup(hostname, options, (err, addresses, maybeFamily) => {
if (err) {
callback(err, addresses, maybeFamily)
return
}
const toValidate = Array.isArray(addresses) ? addresses : [{ address: addresses }]
for (const record of toValidate) {
if (isDisallowedIP(record.address)) {
callback(new Error(FORBIDDEN_IP_ADDRESS), addresses, maybeFamily)
return
}
}
callback(err, addresses, maybeFamily)
})
}
class HttpAgent extends http.Agent {
createConnection (options, callback) {
if (ipaddr.isValid(options.host) && isDisallowedIP(options.host)) {
callback(new Error(FORBIDDEN_IP_ADDRESS))
return undefined
}
// @ts-ignore
return super.createConnection({ ...options, lookup: dnsLookup }, callback)
}
}
class HttpsAgent extends https.Agent {
createConnection (options, callback) {
if (ipaddr.isValid(options.host) && isDisallowedIP(options.host)) {
callback(new Error(FORBIDDEN_IP_ADDRESS))
return undefined
}
// @ts-ignore
return super.createConnection({ ...options, lookup: dnsLookup }, callback)
}
}
/**
* Returns http Agent that will prevent requests to private IPs (to preven SSRF)
*
* @param {string} protocol http or http: or https: or https protocol needed for the request
* @param {boolean} blockPrivateIPs if set to false, this protection will be disabled
*/
module.exports.getProtectedHttpAgent = (protocol, blockPrivateIPs) => {
if (blockPrivateIPs) {
return protocol.startsWith('https') ? HttpsAgent : HttpAgent
}
return protocol.startsWith('https') ? https.Agent : http.Agent
}
/**
* Gets the size and content type of a url's content
*
* @param {string} url
* @param {boolean} blockLocalIPs
* @returns {Promise<{type: string, size: number}>}
*/
exports.getURLMeta = async (url, blockLocalIPs = false) => {
const requestWithMethod = async (method) => new Promise((resolve, reject) => {
const opts = {
uri: url,
method,
followRedirect: exports.getRedirectEvaluator(url, blockLocalIPs),
agentClass: exports.getProtectedHttpAgent((new URL(url)).protocol, blockLocalIPs),
}
const req = request(opts, (err) => {
if (err) reject(err)
})
req.on('response', (response) => {
// Can be undefined for unknown length URLs, e.g. transfer-encoding: chunked
const contentLength = parseInt(response.headers['content-length'], 10)
// No need to get the rest of the response, as we only want header (not really relevant for HEAD, but why not)
req.abort()
resolve({
type: response.headers['content-type'],
size: Number.isNaN(contentLength) ? null : contentLength,
statusCode: response.statusCode,
})
})
})
// We prefer to use a HEAD request, as it doesn't download the content. If the URL doesn't
// support HEAD, or doesn't follow the spec and provide the correct Content-Length, we
// fallback to GET.
let urlMeta = await requestWithMethod('HEAD')
// If HTTP error response, we retry with GET, which may work on non-compliant servers
// (e.g. HEAD doesn't work on signed S3 URLs)
// We look for status codes in the 400 and 500 ranges here, as 3xx errors are
// unlikely to have to do with our choice of method
if (urlMeta.statusCode >= 400 || urlMeta.size === 0 || urlMeta.size == null) {
urlMeta = await requestWithMethod('GET')
}
if (urlMeta.statusCode >= 300) {
// @todo possibly set a status code in the error object to get a more helpful
// hint at what the cause of error is.
throw new Error(`URL server responded with status: ${urlMeta.statusCode}`)
}
const { size, type } = urlMeta
return { size, type }
}