/
fetch_module.js
286 lines (272 loc) Β· 7.82 KB
/
fetch_module.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
'use strict';
const {
ArrayPrototypePush,
Promise,
PromisePrototypeThen,
PromiseResolve,
SafeMap,
StringPrototypeEndsWith,
StringPrototypeSlice,
StringPrototypeStartsWith,
} = primordials;
const {
Buffer: {
concat: BufferConcat
}
} = require('buffer');
const {
ERR_NETWORK_IMPORT_DISALLOWED,
ERR_NETWORK_IMPORT_BAD_RESPONSE,
} = require('internal/errors').codes;
const { URL } = require('internal/url');
const net = require('net');
const path = require('path');
/**
* @typedef CacheEntry
* @property {Promise<string> | string} resolvedHREF
* @property {Record<string, string>} headers
* @property {Promise<Buffer> | Buffer} body
*/
/**
* Only for GET requests, other requests would need new Map
* HTTP cache semantics keep diff caches
*
* Maps HREF to pending cache entry
* @type {Map<string, Promise<CacheEntry> | CacheEntry>}
*/
const cacheForGET = new SafeMap();
// [1] The V8 snapshot doesn't like some C++ APIs to be loaded eagerly. Do it
// lazily/at runtime and not top level of an internal module.
// [2] Creating a new agent instead of using the gloabl agent improves
// performance and precludes the agent becoming tainted.
let HTTPSAgent;
function HTTPSGet(url, opts) {
const https = require('https'); // [1]
HTTPSAgent ??= new https.Agent({ // [2]
keepAlive: true
});
return https.get(url, {
agent: HTTPSAgent,
...opts
});
}
let HTTPAgent;
function HTTPGet(url, opts) {
const http = require('http'); // [1]
HTTPAgent ??= new http.Agent({ // [2]
keepAlive: true
});
return http.get(url, {
agent: HTTPAgent,
...opts
});
}
function dnsLookup(name, opts) {
// eslint-disable-next-line no-func-assign
dnsLookup = require('dns/promises').lookup;
return dnsLookup(name, opts);
}
let zlib;
function createBrotliDecompress() {
zlib ??= require('zlib'); // [1]
// eslint-disable-next-line no-func-assign
createBrotliDecompress = zlib.createBrotliDecompress;
return createBrotliDecompress();
}
function createUnzip() {
zlib ??= require('zlib'); // [1]
// eslint-disable-next-line no-func-assign
createUnzip = zlib.createUnzip;
return createUnzip();
}
/**
* @param {URL} parsed
* @returns {Promise<CacheEntry> | CacheEntry}
*/
function fetchWithRedirects(parsed) {
const existing = cacheForGET.get(parsed.href);
if (existing) {
return existing;
}
const handler = parsed.protocol === 'http:' ? HTTPGet : HTTPSGet;
const result = new Promise((fulfill, reject) => {
const req = handler(parsed, {
headers: {
Accept: '*/*'
}
})
.on('error', reject)
.on('response', (res) => {
function dispose() {
req.destroy();
res.destroy();
}
if (res.statusCode >= 300 && res.statusCode <= 303) {
if (res.headers.location) {
dispose();
try {
const location = new URL(res.headers.location, parsed);
if (location.protocol !== 'http:' &&
location.protocol !== 'https:') {
reject(new ERR_NETWORK_IMPORT_DISALLOWED(
res.headers.location,
parsed.href,
'cannot redirect to non-network location'));
return;
}
return PromisePrototypeThen(
PromiseResolve(fetchWithRedirects(location)),
(entry) => {
cacheForGET.set(parsed.href, entry);
fulfill(entry);
});
} catch (e) {
dispose();
reject(e);
}
}
}
if (res.statusCode > 303 || res.statusCode < 200) {
dispose();
reject(
new ERR_NETWORK_IMPORT_BAD_RESPONSE(
parsed.href,
'HTTP response returned status code of ' + res.statusCode));
return;
}
const { headers } = res;
const contentType = headers['content-type'];
if (!contentType) {
dispose();
reject(new ERR_NETWORK_IMPORT_BAD_RESPONSE(
parsed.href,
'the \'Content-Type\' header is required'));
return;
}
/**
* @type {CacheEntry}
*/
const entry = {
resolvedHREF: parsed.href,
headers: {
'content-type': res.headers['content-type']
},
body: new Promise((f, r) => {
const buffers = [];
let size = 0;
let bodyStream = res;
let onError;
if (res.headers['content-encoding'] === 'br') {
bodyStream = createBrotliDecompress();
onError = function onError(error) {
bodyStream.close();
dispose();
reject(error);
r(error);
};
res.on('error', onError);
res.pipe(bodyStream);
} else if (res.headers['content-encoding'] === 'gzip' ||
res.headers['content-encoding'] === 'deflate') {
bodyStream = createUnzip();
onError = function onError(error) {
bodyStream.close();
dispose();
reject(error);
r(error);
};
res.on('error', onError);
res.pipe(bodyStream);
} else {
onError = function onError(error) {
dispose();
reject(error);
r(error);
};
}
bodyStream.on('error', onError);
bodyStream.on('data', (d) => {
ArrayPrototypePush(buffers, d);
size += d.length;
});
bodyStream.on('end', () => {
const body = entry.body = /** @type {Buffer} */(
BufferConcat(buffers, size)
);
f(body);
});
}),
};
cacheForGET.set(parsed.href, entry);
fulfill(entry);
});
});
cacheForGET.set(parsed.href, result);
return result;
}
const allowList = new net.BlockList();
allowList.addAddress('::1', 'ipv6');
allowList.addRange('127.0.0.1', '127.255.255.255');
/**
* Returns if an address has local status by if it is going to a local
* interface or is an address resolved by DNS to be a local interface
* @param {string} hostname url.hostname to test
* @returns {Promise<boolean>}
*/
async function isLocalAddress(hostname) {
try {
if (StringPrototypeStartsWith(hostname, '[') &&
StringPrototypeEndsWith(hostname, ']')) {
hostname = StringPrototypeSlice(hostname, 1, -1);
}
const addr = await dnsLookup(hostname, { verbatim: true });
const ipv = addr.family === 4 ? 'ipv4' : 'ipv6';
return allowList.check(addr.address, ipv);
} catch {
// If it errored, the answer is no.
}
return false;
}
/**
* Fetches a location with a shared cache following redirects.
* Does not respect HTTP cache headers.
*
* This splits the header and body Promises so that things only needing
* headers don't need to wait on the body.
*
* In cases where the request & response have already settled, this returns the
* cache value synchronously.
*
* @param {URL} parsed
* @param {ESModuleContext} context
* @returns {ReturnType<typeof fetchWithRedirects>}
*/
function fetchModule(parsed, { parentURL }) {
const { href } = parsed;
const existing = cacheForGET.get(href);
if (existing) {
return existing;
}
if (parsed.protocol === 'http:') {
return PromisePrototypeThen(isLocalAddress(parsed.hostname), (is) => {
if (is !== true) {
let parent = parentURL;
const parentName = path.basename(parent.pathname);
if (
parentName === '[eval]' ||
parentName === '[stdin]'
) parent = 'command-line';
throw new ERR_NETWORK_IMPORT_DISALLOWED(
href,
parent,
'http can only be used to load local resources (use https instead).'
);
}
return fetchWithRedirects(parsed);
});
}
return fetchWithRedirects(parsed);
}
module.exports = {
fetchModule: fetchModule
};