Allow setting different maxBuffer values for stdout/stderr

index.d.ts

@@ -356,6 +356,10 @@ type StricterOptions<
 	StrictOptions extends CommonOptions,
 > = WideOptions extends StrictOptions ? WideOptions : StrictOptions;
 
+type FdGenericOption<OptionType> = OptionType | {
+	readonly [FdName in FromOption]?: OptionType
+};
+
 type CommonOptions<IsSync extends boolean = boolean> = {
 	/**
 	Prefer locally installed binaries when looking for a binary to execute.
@@ -596,9 +600,11 @@ type CommonOptions<IsSync extends boolean = boolean> = {
 	- If the `lines` option is `true`: in lines.
 	- If a transform in object mode is used: in objects.
 
+	By default, this applies to both `stdout` and `stderr`, but different values can also be passed.
+
 	@default 100_000_000
 	*/
-	readonly maxBuffer?: number;
+	readonly maxBuffer?: FdGenericOption<number>;
 
 	/**
 	Signal used to terminate the subprocess when:
@@ -738,7 +744,32 @@ type CommonOptions<IsSync extends boolean = boolean> = {
 	readonly cancelSignal?: Unless<IsSync, AbortSignal>;
 };
 
+/**
+Subprocess options.
+
+Some options are related to the subprocess output: `maxBuffer`. By default, those options apply to all file descriptors (`stdout`, `stderr`, etc.). A plain object can be passed instead to apply them to only `stdout`, `stderr`, `fd3`, etc.
+
+@example
+
+```
+await execa('./run.js', {maxBuffer: 1e6}) // Same value for stdout and stderr
+await execa('./run.js', {maxBuffer: {stdout: 1e4, stderr: 1e6}}) // Different values
+```
+*/
 export type Options = CommonOptions<false>;
+
+/**
+Subprocess options, with synchronous methods.
+
+Some options are related to the subprocess output: `maxBuffer`. By default, those options apply to all file descriptors (`stdout`, `stderr`, etc.). A plain object can be passed instead to apply them to only `stdout`, `stderr`, `fd3`, etc.
+
+@example
+
+```
+execaSync('./run.js', {maxBuffer: 1e6}) // Same value for stdout and stderr
+execaSync('./run.js', {maxBuffer: {stdout: 1e4, stderr: 1e6}}) // Different values
+```
+*/
 export type SyncOptions = CommonOptions<true>;
 
 declare abstract class CommonResult<

index.test-d.ts

@@ -1469,6 +1469,27 @@ execaSync('unicorns', {encoding: 'ascii'});
 expectError(execa('unicorns', {encoding: 'unknownEncoding'}));
 expectError(execaSync('unicorns', {encoding: 'unknownEncoding'}));
 
+execa('unicorns', {maxBuffer: {}});
+expectError(execa('unicorns', {maxBuffer: []}));
+execa('unicorns', {maxBuffer: {stdout: 0}});
+execa('unicorns', {maxBuffer: {stderr: 0}});
+execa('unicorns', {maxBuffer: {stdout: 0, stderr: 0} as const});
+execa('unicorns', {maxBuffer: {all: 0}});
+execa('unicorns', {maxBuffer: {fd1: 0}});
+execa('unicorns', {maxBuffer: {fd2: 0}});
+execa('unicorns', {maxBuffer: {fd3: 0}});
+expectError(execa('unicorns', {maxBuffer: {stdout: '0'}}));
+execaSync('unicorns', {maxBuffer: {}});
+expectError(execaSync('unicorns', {maxBuffer: []}));
+execaSync('unicorns', {maxBuffer: {stdout: 0}});
+execaSync('unicorns', {maxBuffer: {stderr: 0}});
+execaSync('unicorns', {maxBuffer: {stdout: 0, stderr: 0} as const});
+execaSync('unicorns', {maxBuffer: {all: 0}});
+execaSync('unicorns', {maxBuffer: {fd1: 0}});
+execaSync('unicorns', {maxBuffer: {fd2: 0}});
+execaSync('unicorns', {maxBuffer: {fd3: 0}});
+expectError(execaSync('unicorns', {maxBuffer: {stdout: '0'}}));
+
 expectError(execa('unicorns', {stdio: []}));
 expectError(execaSync('unicorns', {stdio: []}));
 expectError(execa('unicorns', {stdio: ['pipe']}));

lib/arguments/create.js

@@ -60,4 +60,4 @@ const mergeOption = (optionName, boundOptionValue, optionValue) => {
 	return optionValue;
 };
 
-const DEEP_OPTIONS = new Set(['env']);
+const DEEP_OPTIONS = new Set(['env', 'maxBuffer']);

lib/arguments/options.js

@@ -11,6 +11,7 @@ import {validateEncoding, BINARY_ENCODINGS} from './encoding.js';
 import {handleNodeOption} from './node.js';
 import {joinCommand} from './escape.js';
 import {normalizeCwd, normalizeFileUrl} from './cwd.js';
+import {normalizeFdSpecificOptions} from './specific.js';
 
 export const handleCommand = (filePath, rawArgs, rawOptions) => {
 	const startTime = getStartTime();
@@ -26,7 +27,8 @@ export const handleOptions = (filePath, rawArgs, rawOptions) => {
 
 	const {command: file, args, options: initialOptions} = crossSpawn._parse(processedFile, processedArgs, processedOptions);
 
-	const options = addDefaultOptions(initialOptions);
+	const fdOptions = normalizeFdSpecificOptions(initialOptions);
+	const options = addDefaultOptions(fdOptions);
 	validateTimeout(options);
 	validateEncoding(options);
 	options.shell = normalizeFileUrl(options.shell);
@@ -43,7 +45,6 @@ export const handleOptions = (filePath, rawArgs, rawOptions) => {
 };
 
 const addDefaultOptions = ({
-	maxBuffer = DEFAULT_MAX_BUFFER,
 	buffer = true,
 	stripFinalNewline = true,
 	extendEnv = true,
@@ -63,7 +64,6 @@ const addDefaultOptions = ({
 	...options
 }) => ({
 	...options,
-	maxBuffer,
 	buffer,
 	stripFinalNewline,
 	extendEnv,
@@ -82,8 +82,6 @@ const addDefaultOptions = ({
 	serialization,
 });
 
-const DEFAULT_MAX_BUFFER = 1000 * 1000 * 100;
-
 const getEnv = ({env: envOption, extendEnv, preferLocal, node, localDir, nodePath}) => {
 	const env = extendEnv ? {...process.env, ...envOption} : envOption;
 

lib/arguments/specific.js

@@ -0,0 +1,74 @@
+import isPlainObject from 'is-plain-obj';
+import {STANDARD_STREAMS_ALIASES} from '../utils.js';
+
+export const normalizeFdSpecificOptions = options => {
+	const optionBaseArray = Array.from({length: getStdioLength(options)});
+
+	const optionsCopy = {...options};
+	for (const optionName of FD_SPECIFIC_OPTIONS) {
+		const optionArray = normalizeFdSpecificOption(options[optionName], [...optionBaseArray], optionName);
+		optionsCopy[optionName] = addDefaultValue(optionArray, optionName);
+	}
+
+	return optionsCopy;
+};
+
+const getStdioLength = ({stdio}) => Array.isArray(stdio)
+	? Math.max(stdio.length, STANDARD_STREAMS_ALIASES.length)
+	: STANDARD_STREAMS_ALIASES.length;
+
+const FD_SPECIFIC_OPTIONS = ['maxBuffer'];
+
+const normalizeFdSpecificOption = (optionValue, optionArray, optionName) => isPlainObject(optionValue)
+	? normalizeOptionObject(optionValue, optionArray, optionName)
+	: optionArray.fill(optionValue);
+
+const normalizeOptionObject = (optionValue, optionArray, optionName) => {
+	for (const [fdName, fdValue] of Object.entries(optionValue)) {
+		for (const fdNumber of parseFdName(fdName, optionName, optionArray)) {
+			optionArray[fdNumber] = fdValue;
+		}
+	}
+
+	return optionArray;
+};
+
+const parseFdName = (fdName, optionName, optionArray) => {
+	const fdNumber = parseFd(fdName);
+	if (fdNumber === undefined || fdNumber === 0) {
+		throw new TypeError(`"${optionName}.${fdName}" is invalid.
+It must be "${optionName}.stdout", "${optionName}.stderr", "${optionName}.all", or "${optionName}.fd3", "${optionName}.fd4" (and so on).`);
+	}
+
+	if (fdNumber >= optionArray.length) {
+		throw new TypeError(`"${optionName}.${fdName}" is invalid: that file descriptor does not exist.
+Please set the "stdio" option to ensure that file descriptor exists.`);
+	}
+
+	return fdNumber === 'all' ? [1, 2] : [fdNumber];
+};
+
+export const parseFd = fdName => {
+	if (fdName === 'all') {
+		return fdName;
+	}
+
+	if (STANDARD_STREAMS_ALIASES.includes(fdName)) {
+		return STANDARD_STREAMS_ALIASES.indexOf(fdName);
+	}
+
+	const regexpResult = FD_REGEXP.exec(fdName);
+	if (regexpResult !== null) {
+		return Number(regexpResult[1]);
+	}
+};
+
+const FD_REGEXP = /^fd(\d+)$/;
+
+const addDefaultValue = (optionArray, optionName) => optionArray.map(optionValue => optionValue === undefined
+	? DEFAULT_OPTIONS[optionName]
+	: optionValue);
+
+const DEFAULT_OPTIONS = {
+	maxBuffer: 1000 * 1000 * 100,
+};

lib/exit/code.js

@@ -1,4 +1,5 @@
 import {DiscardedError} from '../return/cause.js';
+import {isMaxBufferSync} from '../stream/max-buffer.js';
 
 export const waitForSuccessfulExit = async exitPromise => {
 	const [exitCode, signal] = await exitPromise;
@@ -13,9 +14,7 @@ export const waitForSuccessfulExit = async exitPromise => {
 export const getSyncExitResult = ({error, status: exitCode, signal, output}, {maxBuffer}) => {
 	const resultError = getResultError(error, exitCode, signal);
 	const timedOut = resultError?.code === 'ETIMEDOUT';
-	const isMaxBuffer = resultError?.code === 'ENOBUFS'
-		&& output !== null
-		&& output.some(result => result !== null && result.length > maxBuffer);
+	const isMaxBuffer = isMaxBufferSync(resultError, output, maxBuffer);
 	return {resultError, exitCode, signal, timedOut, isMaxBuffer};
 };
 

lib/pipe/validate.js

@@ -1,7 +1,7 @@
 import {normalizeArguments} from '../arguments/normalize.js';
-import {STANDARD_STREAMS_ALIASES} from '../utils.js';
 import {getStartTime} from '../return/duration.js';
 import {serializeOptionValue} from '../stdio/native.js';
+import {parseFd} from '../arguments/specific.js';
 
 export const normalizePipeArguments = ({source, sourcePromise, boundOptions, createNested}, ...args) => {
 	const startTime = getStartTime();
@@ -114,17 +114,9 @@ const getFdNumber = (fileDescriptors, fdName, isWritable) => {
 };
 
 const parseFdNumber = (fdName, isWritable) => {
-	if (fdName === 'all') {
-		return fdName;
-	}
-
-	if (STANDARD_STREAMS_ALIASES.includes(fdName)) {
-		return STANDARD_STREAMS_ALIASES.indexOf(fdName);
-	}
-
-	const regexpResult = FD_REGEXP.exec(fdName);
-	if (regexpResult !== null) {
-		return Number(regexpResult[1]);
+	const fdNumber = parseFd(fdName);
+	if (fdNumber !== undefined) {
+		return fdNumber;
 	}
 
 	const {validOptions, defaultValue} = isWritable
@@ -135,8 +127,6 @@ It must be ${validOptions} or "fd3", "fd4" (and so on).
 It is optional and defaults to "${defaultValue}".`);
 };
 
-const FD_REGEXP = /^fd(\d+)$/;
-
 const validateFdNumber = (fdNumber, fdName, isWritable, fileDescriptors) => {
 	const fileDescriptor = fileDescriptors[getUsedDescriptor(fdNumber)];
 	if (fileDescriptor === undefined) {

lib/stdio/handle.js

@@ -1,3 +1,4 @@
+import {getStreamName} from '../utils.js';
 import {getStdioItemType, isRegularUrl, isUnknownStdioString, FILE_TYPES} from './type.js';
 import {getStreamDirection} from './direction.js';
 import {normalizeStdio} from './option.js';
@@ -18,20 +19,17 @@ export const handleInput = (addProperties, options, verboseInfo, isSync) => {
 // This is what users would expect.
 // For example, `stdout: ['ignore']` behaves the same as `stdout: 'ignore'`.
 const getFileDescriptor = ({stdioOption, fdNumber, addProperties, options, isSync}) => {
-	const optionName = getOptionName(fdNumber);
+	const optionName = getStreamName(fdNumber);
 	const {stdioItems: initialStdioItems, isStdioArray} = initializeStdioItems({stdioOption, fdNumber, options, optionName});
 	const direction = getStreamDirection(initialStdioItems, fdNumber, optionName);
 	const stdioItems = initialStdioItems.map(stdioItem => handleNativeStream({stdioItem, isStdioArray, fdNumber, direction, isSync}));
 	const normalizedStdioItems = normalizeTransforms(stdioItems, optionName, direction, options);
 	const objectMode = getObjectMode(normalizedStdioItems, direction);
 	validateFileObjectMode(normalizedStdioItems, objectMode);
 	const finalStdioItems = normalizedStdioItems.map(stdioItem => addStreamProperties(stdioItem, addProperties, direction, options));
-	return {streamName: optionName, direction, objectMode, stdioItems: finalStdioItems};
+	return {direction, objectMode, stdioItems: finalStdioItems};
 };
 
-const getOptionName = fdNumber => KNOWN_OPTION_NAMES[fdNumber] ?? `stdio[${fdNumber}]`;
-const KNOWN_OPTION_NAMES = ['stdin', 'stdout', 'stderr'];
-
 const initializeStdioItems = ({stdioOption, fdNumber, options, optionName}) => {
 	const values = Array.isArray(stdioOption) ? stdioOption : [stdioOption];
 	const initialStdioItems = [

lib/stdio/output-sync.js

@@ -1,5 +1,6 @@
 import {writeFileSync} from 'node:fs';
 import {shouldLogOutput, logLinesSync} from '../verbose/output.js';
+import {getMaxBufferSync} from '../stream/max-buffer.js';
 import {joinToString, joinToUint8Array, bufferToUint8Array, isUint8Array, concatUint8Arrays} from './uint-array.js';
 import {getGenerators, runGeneratorsSync} from './generator.js';
 import {splitLinesSync} from './split.js';
@@ -22,7 +23,7 @@ const transformOutputResultSync = ({result, fileDescriptors, fdNumber, state, is
 		return;
 	}
 
-	const truncatedResult = truncateResult(result, isMaxBuffer, maxBuffer);
+	const truncatedResult = truncateResult(result, isMaxBuffer, getMaxBufferSync(maxBuffer));
 	const uint8ArrayResult = bufferToUint8Array(truncatedResult);
 	const {stdioItems, objectMode} = fileDescriptors[fdNumber];
 	const generators = getGenerators(stdioItems);

lib/stream/all.js

@@ -12,7 +12,7 @@ export const waitForAllStream = ({subprocess, encoding, buffer, maxBuffer, lines
 	fdNumber: 1,
 	encoding,
 	buffer,
-	maxBuffer: maxBuffer * 2,
+	maxBuffer: maxBuffer[1] + maxBuffer[2],
 	lines,
 	isAll: true,
 	allMixed: getAllMixed(subprocess),

lib/stream/max-buffer.js

@@ -1,13 +1,19 @@
 import {MaxBufferError} from 'get-stream';
+import {getStreamName} from '../utils.js';
 
-export const handleMaxBuffer = ({error, stream, readableObjectMode, lines, encoding, streamName}) => {
+export const handleMaxBuffer = ({error, stream, readableObjectMode, lines, encoding, fdNumber, isAll}) => {
 	if (!(error instanceof MaxBufferError)) {
-		return;
+		throw error;
+	}
+
+	if (isAll) {
+		return error;
 	}
 
 	const unit = getMaxBufferUnit(readableObjectMode, lines, encoding);
-	error.maxBufferInfo = {unit, streamName};
+	error.maxBufferInfo = {fdNumber, unit};
 	stream.destroy();
+	throw error;
 };
 
 const getMaxBufferUnit = (readableObjectMode, lines, encoding) => {
@@ -27,16 +33,23 @@ const getMaxBufferUnit = (readableObjectMode, lines, encoding) => {
 };
 
 export const getMaxBufferMessage = (error, maxBuffer) => {
-	const {unit, streamName} = getMaxBufferInfo(error);
-	return `Command's ${streamName} was larger than ${maxBuffer} ${unit}`;
+	const {streamName, threshold, unit} = getMaxBufferInfo(error, maxBuffer);
+	return `Command's ${streamName} was larger than ${threshold} ${unit}`;
 };
 
-const getMaxBufferInfo = error => {
+const getMaxBufferInfo = (error, maxBuffer) => {
 	if (error?.maxBufferInfo === undefined) {
-		return {unit: 'bytes', streamName: 'output'};
+		return {streamName: 'output', threshold: maxBuffer[1], unit: 'bytes'};
 	}
 
-	const {maxBufferInfo} = error;
+	const {maxBufferInfo: {fdNumber, unit}} = error;
 	delete error.maxBufferInfo;
-	return maxBufferInfo;
+	return {streamName: getStreamName(fdNumber), threshold: maxBuffer[fdNumber], unit};
 };
+
+export const isMaxBufferSync = (resultError, output, maxBuffer) => resultError?.code === 'ENOBUFS'
+	&& output !== null
+	&& output.some(result => result !== null && result.length > getMaxBufferSync(maxBuffer));
+
+// `spawnSync()` does not allow differentiating `maxBuffer` per file descriptor, so we always use `stdout`
+export const getMaxBufferSync = maxBuffer => maxBuffer[1];

lib/stream/resolve.js

@@ -56,7 +56,7 @@ export const getSubprocessResult = async ({
 
 // Read the contents of `subprocess.std*` and|or wait for its completion
 const waitForSubprocessStreams = ({subprocess, encoding, buffer, maxBuffer, lines, stripFinalNewline, verboseInfo, streamInfo}) =>
-	subprocess.stdio.map((stream, fdNumber) => waitForSubprocessStream({stream, fdNumber, encoding, buffer, maxBuffer, lines, isAll: false, allMixed: false, stripFinalNewline, verboseInfo, streamInfo}));
+	subprocess.stdio.map((stream, fdNumber) => waitForSubprocessStream({stream, fdNumber, encoding, buffer, maxBuffer: maxBuffer[fdNumber], lines, isAll: false, allMixed: false, stripFinalNewline, verboseInfo, streamInfo}));
 
 // Transforms replace `subprocess.std*`, which means they are not exposed to users.
 // However, we still want to wait for their completion.