/
url.js
135 lines (119 loc) · 3.86 KB
/
url.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
const router = require('express').Router
const request = require('request')
const { URL } = require('node:url')
const validator = require('validator')
const { startDownUpload } = require('../helpers/upload')
const { getURLMeta, getRedirectEvaluator, getProtectedHttpAgent } = require('../helpers/request')
const logger = require('../logger')
/**
* Validates that the download URL is secure
*
* @param {string} url the url to validate
* @param {boolean} ignoreTld whether to allow local addresses
*/
const validateURL = (url, ignoreTld) => {
if (!url) {
return false
}
const validURLOpts = {
protocols: ['http', 'https'],
require_protocol: true,
require_tld: !ignoreTld,
}
if (!validator.isURL(url, validURLOpts)) {
return false
}
return true
}
/**
* @callback downloadCallback
* @param {Error} err
* @param {string | Buffer | Buffer[]} chunk
*/
/**
* Downloads the content in the specified url, and passes the data
* to the callback chunk by chunk.
*
* @param {string} url
* @param {boolean} blockLocalIPs
* @param {string} traceId
* @returns {Promise}
*/
const downloadURL = async (url, blockLocalIPs, traceId) => {
const opts = {
uri: url,
method: 'GET',
followRedirect: getRedirectEvaluator(url, blockLocalIPs),
agentClass: getProtectedHttpAgent((new URL(url)).protocol, blockLocalIPs),
}
return new Promise((resolve, reject) => {
const req = request(opts)
.on('response', (resp) => {
if (resp.statusCode >= 300) {
req.abort() // No need to keep request
reject(new Error(`URL server responded with status: ${resp.statusCode}`))
return
}
// Don't allow any more data to flow yet.
// https://github.com/request/request/issues/1990#issuecomment-184712275
resp.pause()
resolve(resp)
})
.on('error', (err) => {
logger.error(err, 'controller.url.download.error', traceId)
reject(err)
})
})
}
/**
* Fteches the size and content type of a URL
*
* @param {object} req expressJS request object
* @param {object} res expressJS response object
*/
const meta = async (req, res) => {
try {
logger.debug('URL file import handler running', null, req.id)
const { allowLocalUrls } = req.companion.options
if (!validateURL(req.body.url, allowLocalUrls)) {
logger.debug('Invalid request body detected. Exiting url meta handler.', null, req.id)
return res.status(400).json({ error: 'Invalid request body' })
}
const urlMeta = await getURLMeta(req.body.url, !allowLocalUrls)
return res.json(urlMeta)
} catch (err) {
logger.error(err, 'controller.url.meta.error', req.id)
return res.status(err.status || 500).json({ message: 'failed to fetch URL metadata' })
}
}
/**
* Handles the reques of import a file from a remote URL, and then
* subsequently uploading it to the specified destination.
*
* @param {object} req expressJS request object
* @param {object} res expressJS response object
*/
const get = async (req, res) => {
logger.debug('URL file import handler running', null, req.id)
const { allowLocalUrls } = req.companion.options
if (!validateURL(req.body.url, allowLocalUrls)) {
logger.debug('Invalid request body detected. Exiting url import handler.', null, req.id)
res.status(400).json({ error: 'Invalid request body' })
return
}
async function getSize () {
const { size } = await getURLMeta(req.body.url, !allowLocalUrls)
return size
}
async function download () {
return downloadURL(req.body.url, !allowLocalUrls, req.id)
}
function onUnhandledError (err) {
logger.error(err, 'controller.url.error', req.id)
res.status(err.status || 500).json({ message: 'failed to fetch URL metadata' })
}
startDownUpload({ req, res, getSize, download, onUnhandledError })
}
module.exports = () => router()
.post('/meta', meta)
.post('/get', get)