From 6ac574e00a06731288347acea1e8246091196953 Mon Sep 17 00:00:00 2001 From: Dmitriy Mozgovoy Date: Tue, 31 Jan 2023 01:10:39 +0200 Subject: [PATCH] feat(fomdata): added support for spec-compliant FormData & Blob types; (#5316) --- .eslintrc.cjs | 4 +- lib/adapters/http.js | 26 +++++- lib/env/classes/FormData.js | 4 +- lib/helpers/formDataToStream.js | 110 +++++++++++++++++++++++++ lib/helpers/readBlob.js | 15 ++++ lib/helpers/toFormData.js | 18 +--- lib/utils.js | 36 +++++++- package.json | 6 +- test/unit/adapters/http.js | 140 +++++++++++++++++++++++++------- 9 files changed, 304 insertions(+), 55 deletions(-) create mode 100644 lib/helpers/formDataToStream.js create mode 100644 lib/helpers/readBlob.js diff --git a/.eslintrc.cjs b/.eslintrc.cjs index a6a4cc6614..11c8ad99db 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -1,12 +1,12 @@ module.exports = { "env": { "browser": true, - "es2017": true, + "es2018": true, "node": true }, "extends": "eslint:recommended", "parserOptions": { - "ecmaVersion": 2017, + "ecmaVersion": 2018, "sourceType": "module" }, "rules": { diff --git a/lib/adapters/http.js b/lib/adapters/http.js index 2d9c5e12a6..17e3a1de9d 100755 --- a/lib/adapters/http.js +++ b/lib/adapters/http.js @@ -19,6 +19,8 @@ import stream from 'stream'; import AxiosHeaders from '../core/AxiosHeaders.js'; import AxiosTransformStream from '../helpers/AxiosTransformStream.js'; import EventEmitter from 'events'; +import formDataToStream from "../helpers/formDataToStream.js"; +import readBlob from "../helpers/readBlob.js"; import ZlibHeaderTransformStream from '../helpers/ZlibHeaderTransformStream.js'; const zlibOptions = { @@ -243,9 +245,27 @@ export default isHttpAdapterSupported && function httpAdapter(config) { let maxUploadRate = undefined; let maxDownloadRate = undefined; - // support for https://www.npmjs.com/package/form-data api - if (utils.isFormData(data) && utils.isFunction(data.getHeaders)) { + // support for spec compliant FormData objects + if (utils.isSpecCompliantForm(data)) { + const userBoundary = headers.getContentType(/boundary=([-_\w\d]{10,70})/i); + + data = formDataToStream(data, (formHeaders) => { + headers.set(formHeaders); + }, { + tag: `axios-${VERSION}-boundary`, + boundary: userBoundary && userBoundary[1] || undefined + }); + // support for https://www.npmjs.com/package/form-data api + } else if (utils.isFormData(data) && utils.isFunction(data.getHeaders)) { headers.set(data.getHeaders()); + if (utils.isFunction(data.getLengthSync)) { // check if the undocumented API exists + const knownLength = data.getLengthSync(); + !utils.isUndefined(knownLength) && headers.setContentLength(knownLength, false); + } + } else if (utils.isBlob(data)) { + data.size && headers.setContentType(data.type || 'application/octet-stream'); + headers.setContentLength(data.size || 0); + data = stream.Readable.from(readBlob(data)); } else if (data && !utils.isStream(data)) { if (Buffer.isBuffer(data)) { // Nothing to do... @@ -262,7 +282,7 @@ export default isHttpAdapterSupported && function httpAdapter(config) { } // Add Content-Length header if data exists - headers.set('Content-Length', data.length, false); + headers.setContentLength(data.length, false); if (config.maxBodyLength > -1 && data.length > config.maxBodyLength) { return reject(new AxiosError( diff --git a/lib/env/classes/FormData.js b/lib/env/classes/FormData.js index d3a90ec4b0..862adb93c3 100644 --- a/lib/env/classes/FormData.js +++ b/lib/env/classes/FormData.js @@ -1,2 +1,2 @@ -import FormData from 'form-data'; -export default FormData; +import _FormData from 'form-data'; +export default typeof FormData !== 'undefined' ? FormData : _FormData; diff --git a/lib/helpers/formDataToStream.js b/lib/helpers/formDataToStream.js new file mode 100644 index 0000000000..3849935c2f --- /dev/null +++ b/lib/helpers/formDataToStream.js @@ -0,0 +1,110 @@ +import {Readable} from 'stream'; +import utils from "../utils.js"; +import readBlob from "./readBlob.js"; + +const BOUNDARY_ALPHABET = utils.ALPHABET.ALPHA_DIGIT + '-_'; + +const textEncoder = new TextEncoder(); + +const CRLF = '\r\n'; +const CRLF_BYTES = textEncoder.encode(CRLF); +const CRLF_BYTES_COUNT = 2; + +class FormDataPart { + constructor(name, value) { + const {escapeName} = this.constructor; + const isStringValue = utils.isString(value); + + let headers = `Content-Disposition: form-data; name="${escapeName(name)}"${ + !isStringValue && value.name ? `; filename="${escapeName(value.name)}"` : '' + }${CRLF}`; + + if (isStringValue) { + value = textEncoder.encode(String(value).replace(/\r?\n|\r\n?/g, CRLF)); + } else { + headers += `Content-Type: ${value.type || "application/octet-stream"}${CRLF}` + } + + this.headers = textEncoder.encode(headers + CRLF); + + this.contentLength = isStringValue ? value.byteLength : value.size; + + this.size = this.headers.byteLength + this.contentLength + CRLF_BYTES_COUNT; + + this.name = name; + this.value = value; + } + + async *encode(){ + yield this.headers; + + const {value} = this; + + if(utils.isTypedArray(value)) { + yield value; + } else { + yield* readBlob(value); + } + + yield CRLF_BYTES; + } + + static escapeName(name) { + return String(name).replace(/[\r\n"]/g, (match) => ({ + '\r' : '%0D', + '\n' : '%0A', + '"' : '%22', + }[match])); + } +} + +const formDataToStream = (form, headersHandler, options) => { + const { + tag = 'form-data-boundary', + size = 25, + boundary = tag + '-' + utils.generateString(size, BOUNDARY_ALPHABET) + } = options || {}; + + if(!utils.isFormData(form)) { + throw TypeError('FormData instance required'); + } + + if (boundary.length < 1 || boundary.length > 70) { + throw Error('boundary must be 10-70 characters long') + } + + const boundaryBytes = textEncoder.encode('--' + boundary + CRLF); + const footerBytes = textEncoder.encode('--' + boundary + '--' + CRLF + CRLF); + let contentLength = footerBytes.byteLength; + + const parts = Array.from(form.entries()).map(([name, value]) => { + const part = new FormDataPart(name, value); + contentLength += part.size; + return part; + }); + + contentLength += boundaryBytes.byteLength * parts.length; + + contentLength = utils.toFiniteNumber(contentLength); + + const computedHeaders = { + 'Content-Type': `multipart/form-data; boundary=${boundary}` + } + + if (Number.isFinite(contentLength)) { + computedHeaders['Content-Length'] = contentLength; + } + + headersHandler && headersHandler(computedHeaders); + + return Readable.from((async function *() { + for(const part of parts) { + yield boundaryBytes; + yield* part.encode(); + } + + yield footerBytes; + })()); +}; + +export default formDataToStream; diff --git a/lib/helpers/readBlob.js b/lib/helpers/readBlob.js new file mode 100644 index 0000000000..19f147681e --- /dev/null +++ b/lib/helpers/readBlob.js @@ -0,0 +1,15 @@ +const {asyncIterator} = Symbol; + +const readBlob = async function* (blob) { + if (blob.stream) { + yield* blob.stream() + } else if (blob.arrayBuffer) { + yield await blob.arrayBuffer() + } else if (blob[asyncIterator]) { + yield* asyncIterator.call(blob); + } else { + yield blob; + } +} + +export default readBlob; diff --git a/lib/helpers/toFormData.js b/lib/helpers/toFormData.js index ce99ce15c7..327a979284 100644 --- a/lib/helpers/toFormData.js +++ b/lib/helpers/toFormData.js @@ -2,7 +2,8 @@ import utils from '../utils.js'; import AxiosError from '../core/AxiosError.js'; -import envFormData from '../env/classes/FormData.js'; +// temporary hotfix to avoid circular references until AxiosURLSearchParams is refactored +import PlatformFormData from '../platform/node/classes/FormData.js'; /** * Determines if the given thing is a array or js object. @@ -59,17 +60,6 @@ const predicates = utils.toFlatObject(utils, {}, null, function filter(prop) { return /^is[A-Z]/.test(prop); }); -/** - * If the thing is a FormData object, return true, otherwise return false. - * - * @param {unknown} thing - The thing to check. - * - * @returns {boolean} - */ -function isSpecCompliant(thing) { - return thing && utils.isFunction(thing.append) && thing[Symbol.toStringTag] === 'FormData' && thing[Symbol.iterator]; -} - /** * Convert a data object to FormData * @@ -99,7 +89,7 @@ function toFormData(obj, formData, options) { } // eslint-disable-next-line no-param-reassign - formData = formData || new (envFormData || FormData)(); + formData = formData || new (PlatformFormData || FormData)(); // eslint-disable-next-line no-param-reassign options = utils.toFlatObject(options, { @@ -117,7 +107,7 @@ function toFormData(obj, formData, options) { const dots = options.dots; const indexes = options.indexes; const _Blob = options.Blob || typeof Blob !== 'undefined' && Blob; - const useBlob = _Blob && isSpecCompliant(formData); + const useBlob = _Blob && utils.isSpecCompliantForm(formData); if (!utils.isFunction(visitor)) { throw new TypeError('visitor must be a function'); diff --git a/lib/utils.js b/lib/utils.js index 68d791102b..6a388721c1 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -512,7 +512,7 @@ const matchAll = (regExp, str) => { const isHTMLForm = kindOfTest('HTMLFormElement'); const toCamelCase = str => { - return str.toLowerCase().replace(/[_-\s]([a-z\d])(\w*)/g, + return str.toLowerCase().replace(/[-_\s]([a-z\d])(\w*)/g, function replacer(m, p1, p2) { return p1.toUpperCase() + p2; } @@ -596,6 +596,37 @@ const toFiniteNumber = (value, defaultValue) => { return Number.isFinite(value) ? value : defaultValue; } +const ALPHA = 'abcdefghijklmnopqrstuvwxyz' + +const DIGIT = '0123456789'; + +const ALPHABET = { + DIGIT, + ALPHA, + ALPHA_DIGIT: ALPHA + ALPHA.toUpperCase() + DIGIT +} + +const generateString = (size = 16, alphabet = ALPHABET.ALPHA_DIGIT) => { + let str = ''; + const {length} = alphabet; + while (size--) { + str += alphabet[Math.random() * length|0] + } + + return str; +} + +/** + * If the thing is a FormData object, return true, otherwise return false. + * + * @param {unknown} thing - The thing to check. + * + * @returns {boolean} + */ +function isSpecCompliantForm(thing) { + return !!(thing && isFunction(thing.append) && thing[Symbol.toStringTag] === 'FormData' && thing[Symbol.iterator]); +} + const toJSONObject = (obj) => { const stack = new Array(10); @@ -673,5 +704,8 @@ export default { findKey, global: _global, isContextDefined, + ALPHABET, + generateString, + isSpecCompliantForm, toJSONObject }; diff --git a/package.json b/package.json index ee98845f50..17d44cbaf0 100644 --- a/package.json +++ b/package.json @@ -89,6 +89,7 @@ "es6-promise": "^4.2.8", "eslint": "^8.17.0", "express": "^4.18.1", + "formdata-node": "^5.0.0", "formidable": "^2.0.1", "fs-extra": "^10.1.0", "get-stream": "^3.0.0", @@ -126,7 +127,8 @@ }, "browser": { "./lib/adapters/http.js": "./lib/helpers/null.js", - "./lib/platform/node/index.js": "./lib/platform/browser/index.js" + "./lib/platform/node/index.js": "./lib/platform/browser/index.js", + "./lib/platform/node/classes/FormData.js": "./lib/helpers/null.js" }, "jsdelivr": "dist/axios.min.js", "unpkg": "dist/axios.min.js", @@ -201,4 +203,4 @@ "@commitlint/config-conventional" ] } -} \ No newline at end of file +} diff --git a/test/unit/adapters/http.js b/test/unit/adapters/http.js index 9d44c48af1..23faa3b92d 100644 --- a/test/unit/adapters/http.js +++ b/test/unit/adapters/http.js @@ -11,7 +11,7 @@ import fs from 'fs'; import path from 'path'; let server, proxy; import AxiosError from '../../../lib/core/AxiosError.js'; -import FormData from 'form-data'; +import FormDataLegacy from 'form-data'; import formidable from 'formidable'; import express from 'express'; import multer from 'multer'; @@ -21,6 +21,11 @@ import {Throttle} from 'stream-throttle'; import devNull from 'dev-null'; import {AbortController} from 'abortcontroller-polyfill/dist/cjs-ponyfill.js'; import {__setProxy} from "../../../lib/adapters/http.js"; +import {FormData as FormDataPolyfill, Blob as BlobPolyfill, File as FilePolyfill} from 'formdata-node'; + +const FormDataSpecCompliant = typeof FormData !== 'undefined' ? FormData : FormDataPolyfill; +const BlobSpecCompliant = typeof Blob !== 'undefined' ? Blob : BlobPolyfill; +const FileSpecCompliant = typeof File !== 'undefined' ? File : FilePolyfill; const __filename = url.fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -89,6 +94,20 @@ function startHTTPServer(options) { }); } +const handleFormData = (req) => { + return new Promise((resolve, reject) => { + const form = new formidable.IncomingForm(); + + form.parse(req, (err, fields, files) => { + if (err) { + return reject(err); + } + + resolve({fields, files}); + }); + }); +} + function generateReadableStream(length = 1024 * 1024, chunkSize = 10 * 1024, sleep = 50) { return stream.Readable.from(async function* (){ let dataLength = 0; @@ -1553,44 +1572,85 @@ describe('supports http with nodejs', function () { }); describe('FormData', function () { - it('should allow passing FormData', function (done) { - var form = new FormData(); - var file1 = Buffer.from('foo', 'utf8'); - - form.append('foo', "bar"); - form.append('file1', file1, { - filename: 'bar.jpg', - filepath: 'temp/bar.jpg', - contentType: 'image/jpeg' - }); + describe('form-data instance (https://www.npmjs.com/package/form-data)', (done) => { + it('should allow passing FormData', function (done) { + var form = new FormDataLegacy(); + var file1 = Buffer.from('foo', 'utf8'); + + form.append('foo', "bar"); + form.append('file1', file1, { + filename: 'bar.jpg', + filepath: 'temp/bar.jpg', + contentType: 'image/jpeg' + }); - server = http.createServer(function (req, res) { - var receivedForm = new formidable.IncomingForm(); + server = http.createServer(function (req, res) { + var receivedForm = new formidable.IncomingForm(); - receivedForm.parse(req, function (err, fields, files) { - if (err) { - return done(err); - } + receivedForm.parse(req, function (err, fields, files) { + if (err) { + return done(err); + } + + res.end(JSON.stringify({ + fields: fields, + files: files + })); + }); + }).listen(4444, function () { + axios.post('http://localhost:4444/', form, { + headers: { + 'Content-Type': 'multipart/form-data' + } + }).then(function (res) { + assert.deepStrictEqual(res.data.fields, {foo: 'bar'}); + + assert.strictEqual(res.data.files.file1.mimetype, 'image/jpeg'); + assert.strictEqual(res.data.files.file1.originalFilename, 'temp/bar.jpg'); + assert.strictEqual(res.data.files.file1.size, 3); + + done(); + }).catch(done); + }); + }); + }); + + describe('SpecCompliant FormData', (done) => { + it('should allow passing FormData', async function () { + server = await startHTTPServer(async (req, res) => { + const {fields, files} = await handleFormData(req); res.end(JSON.stringify({ - fields: fields, - files: files + fields, + files })); }); - }).listen(4444, function () { - axios.post('http://localhost:4444/', form, { - headers: { - 'Content-Type': 'multipart/form-data' - } - }).then(function (res) { - assert.deepStrictEqual(res.data.fields, {foo: 'bar'}); - assert.strictEqual(res.data.files.file1.mimetype, 'image/jpeg'); - assert.strictEqual(res.data.files.file1.originalFilename, 'temp/bar.jpg'); - assert.strictEqual(res.data.files.file1.size, 3); + const form = new FormDataSpecCompliant(); - done(); - }).catch(done); + const blobContent = 'blob-content'; + + const blob = new BlobSpecCompliant([blobContent], {type: 'image/jpeg'}) + + form.append('foo1', 'bar1'); + form.append('foo2', 'bar2'); + + form.append('file1', blob); + + const {data} = await axios.post(LOCAL_SERVER_URL, form, { + maxRedirects: 0 + }); + + assert.deepStrictEqual(data.fields, {foo1: 'bar1' ,'foo2': 'bar2'}); + + assert.deepStrictEqual(typeof data.files.file1, 'object'); + + const {size, mimetype, originalFilename} = data.files.file1; + + assert.deepStrictEqual( + {size, mimetype, originalFilename}, + {mimetype: 'image/jpeg', originalFilename: 'blob', size: Buffer.from(blobContent).byteLength} + ); }); }); @@ -1633,6 +1693,24 @@ describe('supports http with nodejs', function () { }); }); + describe('Blob', function () { + it('should support Blob', async () => { + server = await startHTTPServer(async (req, res) => { + res.end(await getStream(req)); + }); + + const blobContent = 'blob-content'; + + const blob = new BlobSpecCompliant([blobContent], {type: 'image/jpeg'}) + + const {data} = await axios.post(LOCAL_SERVER_URL, blob, { + maxRedirects: 0 + }); + + assert.deepStrictEqual(data, blobContent); + }); + }); + describe('URLEncoded Form', function () { it('should post object data as url-encoded form if content-type is application/x-www-form-urlencoded', function (done) { var app = express();