diff --git a/src/server/Uploader.js b/src/server/Uploader.js index fb333d2c32..a56e495a63 100644 --- a/src/server/Uploader.js +++ b/src/server/Uploader.js @@ -16,6 +16,9 @@ const logger = require('./logger') const headerSanitize = require('./header-blacklist') const redis = require('./redis') +// Need to limit length or we can get +// "MetadataTooLarge: Your metadata headers exceed the maximum allowed metadata size" in tus / S3 +const MAX_FILENAME_LENGTH = 500 const DEFAULT_FIELD_NAME = 'files[]' const PROTOCOLS = Object.freeze({ multipart: 'multipart', @@ -58,7 +61,9 @@ class Uploader { this.path = `${this.options.pathPrefix}/${Uploader.FILE_NAME_PREFIX}-${this.token}` this.options.metadata = this.options.metadata || {} this.options.fieldname = this.options.fieldname || DEFAULT_FIELD_NAME - this.uploadFileName = this.options.metadata.name || path.basename(this.path) + this.uploadFileName = this.options.metadata.name + ? this.options.metadata.name.substring(0, MAX_FILENAME_LENGTH) + : path.basename(this.path) this.streamsEnded = false this.uploadStopped = false this.writeStream = fs.createWriteStream(this.path, { mode: 0o666 }) // no executable files diff --git a/src/server/controllers/url.js b/src/server/controllers/url.js index 186610dd23..b5c33ee315 100644 --- a/src/server/controllers/url.js +++ b/src/server/controllers/url.js @@ -2,81 +2,11 @@ const router = require('express').Router const request = require('request') const { URL } = require('url') const validator = require('validator') + const Uploader = require('../Uploader') -const reqUtil = require('../helpers/request') +const { getURLMeta, getRedirectEvaluator, getProtectedHttpAgent } = require('../helpers/request') const logger = require('../logger') -module.exports = () => { - return router() - .post('/meta', meta) - .post('/get', get) -} - -/** - * Fteches the size and content type of a URL - * - * @param {object} req expressJS request object - * @param {object} res expressJS response object - */ -const meta = (req, res) => { - logger.debug('URL file import handler running', null, req.id) - const { debug } = req.companion.options - if (!validateURL(req.body.url, debug)) { - logger.debug('Invalid request body detected. Exiting url meta handler.', null, req.id) - return res.status(400).json({ error: 'Invalid request body' }) - } - - reqUtil.getURLMeta(req.body.url, !debug) - .then((meta) => res.json(meta)) - .catch((err) => { - logger.error(err, 'controller.url.meta.error', req.id) - // @todo send more meaningful error message and status code to client if possible - return res.status(err.status || 500).json({ message: 'failed to fetch URL metadata' }) - }) -} - -/** - * Handles the reques of import a file from a remote URL, and then - * subsequently uploading it to the specified destination. - * - * @param {object} req expressJS request object - * @param {object} res expressJS response object - */ -const get = (req, res) => { - logger.debug('URL file import handler running', null, req.id) - const { debug } = req.companion.options - if (!validateURL(req.body.url, debug)) { - logger.debug('Invalid request body detected. Exiting url import handler.', null, req.id) - return res.status(400).json({ error: 'Invalid request body' }) - } - - reqUtil.getURLMeta(req.body.url, !debug) - .then(({ size }) => { - // @ts-ignore - logger.debug('Instantiating uploader.', null, req.id) - const uploader = new Uploader(Uploader.reqToOptions(req, size)) - - if (uploader.hasError()) { - const response = uploader.getResponse() - res.status(response.status).json(response.body) - return - } - - logger.debug('Waiting for socket connection before beginning remote download.', null, req.id) - uploader.onSocketReady(() => { - logger.debug('Socket connection received. Starting remote download.', null, req.id) - downloadURL(req.body.url, uploader.handleChunk.bind(uploader), !debug, req.id) - }) - - const response = uploader.getResponse() - res.status(response.status).json(response.body) - }).catch((err) => { - logger.error(err, 'controller.url.get.error', req.id) - // @todo send more meaningful error message and status code to client if possible - return res.status(err.status || 500).json({ message: 'failed to fetch URL metadata' }) - }) -} - /** * Validates that the download URL is secure * @@ -113,14 +43,14 @@ const validateURL = (url, debug) => { * @param {string} url * @param {downloadCallback} onDataChunk * @param {boolean} blockLocalIPs - * @param {string=} traceId + * @param {string} traceId */ const downloadURL = (url, onDataChunk, blockLocalIPs, traceId) => { const opts = { uri: url, method: 'GET', - followRedirect: reqUtil.getRedirectEvaluator(url, blockLocalIPs), - agentClass: reqUtil.getProtectedHttpAgent((new URL(url)).protocol, blockLocalIPs), + followRedirect: getRedirectEvaluator(url, blockLocalIPs), + agentClass: getProtectedHttpAgent((new URL(url)).protocol, blockLocalIPs), } request(opts) @@ -138,3 +68,77 @@ const downloadURL = (url, onDataChunk, blockLocalIPs, traceId) => { onDataChunk(err, null) }) } + +/** + * Fteches the size and content type of a URL + * + * @param {object} req expressJS request object + * @param {object} res expressJS response object + */ +const meta = async (req, res) => { + try { + logger.debug('URL file import handler running', null, req.id) + const { debug } = req.companion.options + if (!validateURL(req.body.url, debug)) { + logger.debug('Invalid request body detected. Exiting url meta handler.', null, req.id) + return res.status(400).json({ error: 'Invalid request body' }) + } + + const urlMeta = await getURLMeta(req.body.url, !debug) + return res.json(urlMeta) + } catch (err) { + logger.error(err, 'controller.url.meta.error', req.id) + // @todo send more meaningful error message and status code to client if possible + return res.status(err.status || 500).json({ message: 'failed to fetch URL metadata' }) + } +} + +/** + * Handles the reques of import a file from a remote URL, and then + * subsequently uploading it to the specified destination. + * + * @param {object} req expressJS request object + * @param {object} res expressJS response object + */ +const get = async (req, res) => { + try { + logger.debug('URL file import handler running', null, req.id) + const { debug } = req.companion.options + if (!validateURL(req.body.url, debug)) { + logger.debug('Invalid request body detected. Exiting url import handler.', null, req.id) + res.status(400).json({ error: 'Invalid request body' }) + return + } + + const { size } = await getURLMeta(req.body.url, !debug) + + // @ts-ignore + logger.debug('Instantiating uploader.', null, req.id) + const uploader = new Uploader(Uploader.reqToOptions(req, size)) + + if (uploader.hasError()) { + const response = uploader.getResponse() + res.status(response.status).json(response.body) + return + } + + logger.debug('Waiting for socket connection before beginning remote download.', null, req.id) + uploader.onSocketReady(() => { + logger.debug('Socket connection received. Starting remote download.', null, req.id) + downloadURL(req.body.url, uploader.handleChunk.bind(uploader), !debug, req.id) + }) + + const response = uploader.getResponse() + + // NOTE: Uploader will continue running after the http request is responded + res.status(response.status).json(response.body) + } catch (err) { + logger.error(err, 'controller.url.get.error', req.id) + // @todo send more meaningful error message and status code to client if possible + res.status(err.status || 500).json({ message: 'failed to fetch URL metadata' }) + } +} + +module.exports = () => router() + .post('/meta', meta) + .post('/get', get) diff --git a/src/server/helpers/request.js b/src/server/helpers/request.js index 3d7e32d9ce..da312cd962 100644 --- a/src/server/helpers/request.js +++ b/src/server/helpers/request.js @@ -106,20 +106,6 @@ module.exports.getRedirectEvaluator = (rawRequestURL, blockPrivateIPs) => { } } -/** - * Returns http Agent that will prevent requests to private IPs (to preven SSRF) - * - * @param {string} protocol http or http: or https: or https protocol needed for the request - * @param {boolean} blockPrivateIPs if set to false, this protection will be disabled - */ -module.exports.getProtectedHttpAgent = (protocol, blockPrivateIPs) => { - if (blockPrivateIPs) { - return protocol.startsWith('https') ? HttpsAgent : HttpAgent - } - - return protocol.startsWith('https') ? https.Agent : http.Agent -} - function dnsLookup (hostname, options, callback) { dns.lookup(hostname, options, (err, addresses, maybeFamily) => { if (err) { @@ -141,54 +127,69 @@ function dnsLookup (hostname, options, callback) { class HttpAgent extends http.Agent { createConnection (options, callback) { - options.lookup = dnsLookup if (isIPAddress(options.host) && isPrivateIP(options.host)) { callback(new Error(FORBIDDEN_IP_ADDRESS)) - return + return undefined } // @ts-ignore - return super.createConnection(options, callback) + return super.createConnection({ ...options, lookup: dnsLookup }, callback) } } class HttpsAgent extends https.Agent { createConnection (options, callback) { - options.lookup = dnsLookup if (isIPAddress(options.host) && isPrivateIP(options.host)) { callback(new Error(FORBIDDEN_IP_ADDRESS)) - return + return undefined } // @ts-ignore - return super.createConnection(options, callback) + return super.createConnection({ ...options, lookup: dnsLookup }, callback) } } +/** + * Returns http Agent that will prevent requests to private IPs (to preven SSRF) + * + * @param {string} protocol http or http: or https: or https protocol needed for the request + * @param {boolean} blockPrivateIPs if set to false, this protection will be disabled + */ +module.exports.getProtectedHttpAgent = (protocol, blockPrivateIPs) => { + if (blockPrivateIPs) { + return protocol.startsWith('https') ? HttpsAgent : HttpAgent + } + + return protocol.startsWith('https') ? https.Agent : http.Agent +} + /** * Gets the size and content type of a url's content * * @param {string} url - * @param {boolean=} blockLocalIPs + * @param {boolean} blockLocalIPs * @returns {Promise<{type: string, size: number}>} */ exports.getURLMeta = (url, blockLocalIPs = false) => { return new Promise((resolve, reject) => { const opts = { uri: url, - method: 'HEAD', + method: 'GET', followRedirect: exports.getRedirectEvaluator(url, blockLocalIPs), agentClass: exports.getProtectedHttpAgent((new URL(url)).protocol, blockLocalIPs), } - request(opts, (err, response) => { - if (err || response.statusCode >= 300) { + const req = request(opts, (err) => { + if (err) reject(err) + }) + req.on('response', (response) => { + if (response.statusCode >= 300) { // @todo possibly set a status code in the error object to get a more helpful // hint at what the cause of error is. - err = err || new Error(`URL server responded with status: ${response.statusCode}`) - reject(err) + reject(new Error(`URL server responded with status: ${response.statusCode}`)) } else { + req.abort() // No need to get the rest of the response, as we only want header resolve({ type: response.headers['content-type'], - size: parseInt(response.headers['content-length']), + size: parseInt(response.headers['content-length'], 10), }) } })