Merge pull request #1349 from remotion-dev/lambda-overwrite-option

packages/cli/src/config/overwrite.ts

@@ -1,6 +1,4 @@
-import {RenderInternals} from '@remotion/renderer';
-
-let shouldOverwrite = RenderInternals.DEFAULT_OVERWRITE;
+let shouldOverwrite: boolean | null = null;
 
 export const setOverwriteOutput = (newOverwrite: boolean) => {
 	if (typeof newOverwrite !== 'boolean') {
@@ -14,4 +12,8 @@ export const setOverwriteOutput = (newOverwrite: boolean) => {
 	shouldOverwrite = newOverwrite;
 };
 
-export const getShouldOverwrite = () => shouldOverwrite;
+export const getShouldOverwrite = ({
+	defaultValue,
+}: {
+	defaultValue: boolean;
+}): boolean => shouldOverwrite ?? defaultValue;

packages/cli/src/get-cli-options.ts

@@ -217,7 +217,9 @@ export const getCliOptions = async (options: {
 					isLambda: options.isLambda,
 			  });
 
-	const overwrite = ConfigInternals.getShouldOverwrite();
+	const overwrite = ConfigInternals.getShouldOverwrite({
+		defaultValue: !options.isLambda,
+	});
 	const crf = getAndValidateCrf(shouldOutputImageSequence, codec);
 	const pixelFormat = getAndValidatePixelFormat(codec);
 	const imageFormat = getAndValidateImageFormat({

packages/cli/src/test/overwrite.test.ts

@@ -6,7 +6,7 @@ const invalidOverwrite = 555;
 let defaultOverwriteValue: boolean;
 
 beforeAll(() => {
-	defaultOverwriteValue = getShouldOverwrite();
+	defaultOverwriteValue = getShouldOverwrite({defaultValue: true});
 });
 afterEach(() => {
 	setOverwriteOutput(defaultOverwriteValue);
@@ -23,9 +23,9 @@ test('setOverwriteOutput should NOT throw if image format is a boolean value', (
 	expect(() => setOverwriteOutput(true)).not.toThrow();
 });
 test('getShouldOverwrite should return true by default', () => {
-	expect(getShouldOverwrite()).toEqual(true);
+	expect(getShouldOverwrite({defaultValue: true})).toEqual(true);
 });
 test('setOverwriteOutput should return a boolean value', () => {
 	setOverwriteOutput(false);
-	expect(getShouldOverwrite()).toEqual(false);
+	expect(getShouldOverwrite({defaultValue: true})).toEqual(false);
 });

packages/docs/docs/lambda/cli/render.md

@@ -165,3 +165,11 @@ _available from v3.1_
 ### `--out-name`
 
 The file name of the media output as stored in the S3 bucket. By default, it is `out` plus the appropriate file extension, for example: `out.mp4`. Must match `/([0-9a-zA-Z-!_.*'()]+)/g`.
+
+### `--overwrite`
+
+_available from v3.2.25_
+
+If a custom out name is specified and a file already exists at this key in the S3 bucket, decide whether that file will be deleted before the render begins. Default `false`.
+
+An existing file at the output S3 key will conflict with the render and must be deleted beforehand. If this setting is `false` and a conflict occurs, an error will be thrown.

packages/docs/docs/lambda/rendermediaonlambda.md

@@ -226,6 +226,14 @@ _Available from v3.2.10_
 
 A link to CloudWatch (if you haven't disabled it) that you can visit to see the logs for the render.
 
+### `overwrite`
+
+_available from v3.2.25_
+
+If a custom out name is specified and a file already exists at this key in the S3 bucket, decide whether that file will be deleted before the render begins. Default `false`.
+
+An existing file at the output S3 key will conflict with the render and must be deleted beforehand. If this setting is `false` and a conflict occurs, an error will be thrown.
+
 ## See also
 
 - [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/lambda/src/api/render-media-on-lambda.ts)

packages/lambda/src/api/render-media-on-lambda.ts

@@ -47,6 +47,7 @@ export type RenderMediaOnLambdaInput = {
 	concurrencyPerLambda?: number;
 	downloadBehavior?: DownloadBehavior | null;
 	muted?: boolean;
+	overwrite?: boolean;
 };
 
 export type RenderMediaOnLambdaOutput = {
@@ -101,6 +102,7 @@ export const renderMediaOnLambda = async ({
 	concurrencyPerLambda,
 	downloadBehavior,
 	muted,
+	overwrite,
 }: RenderMediaOnLambdaInput): Promise<RenderMediaOnLambdaOutput> => {
 	const actualCodec = validateLambdaCodec(codec);
 	validateServeUrl(serveUrl);
@@ -140,6 +142,7 @@ export const renderMediaOnLambda = async ({
 				downloadBehavior: downloadBehavior ?? {type: 'play-in-browser'},
 				muted: muted ?? false,
 				version: VERSION,
+				overwrite: overwrite ?? false,
 			},
 			region,
 		});

packages/lambda/src/cli/commands/render/render.ts

@@ -66,6 +66,7 @@ export const renderCommand = async (args: string[]) => {
 		everyNthFrame,
 		numberOfGifLoops,
 		muted,
+		overwrite,
 	} = await CliInternals.getCliOptions({
 		type: 'series',
 		isLambda: true,
@@ -109,6 +110,7 @@ export const renderCommand = async (args: string[]) => {
 		everyNthFrame,
 		concurrencyPerLambda: parsedLambdaCli['concurrency-per-lambda'],
 		muted,
+		overwrite,
 	});
 
 	const totalSteps = outName ? 5 : 4;

packages/lambda/src/functions/helpers/__mocks__/io.ts

@@ -8,6 +8,7 @@ import {
 } from '../../../api/__mocks__/mock-s3';
 import type {
 	lambdaDeleteFile as deleteOriginal,
+	lambdaHeadCommand as headOriginal,
 	lambdaLs as lsOriginal,
 	lambdaReadFile as readOriginal,
 	lambdaWriteFile as writeOriginal,
@@ -61,6 +62,28 @@ export const lambdaDeleteFile: typeof deleteOriginal = ({
 	return Promise.resolve(undefined);
 };
 
+export const lambdaHeadCommand: typeof headOriginal = ({
+	bucketName,
+	key,
+	region,
+}) => {
+	const read = readMockS3File({
+		bucketName,
+		key,
+		region,
+	});
+	if (!read) {
+		const err = new Error('File not found');
+		err.name = 'NotFound';
+		throw err;
+	}
+
+	return Promise.resolve({
+		ContentLength: read.content.toString().length,
+		LastModified: new Date(),
+	});
+};
+
 export const lambdaLs: typeof lsOriginal = (
 	input: LambdaLSInput
 ): LambdaLsReturnType => {

packages/lambda/src/functions/helpers/find-output-file-in-bucket.ts

@@ -1,11 +1,10 @@
-import {HeadObjectCommand} from '@aws-sdk/client-s3';
 import type {AwsRegion} from '../..';
 import {ROLE_NAME} from '../../api/iam-validation/suggested-policy';
 import type {CustomCredentials} from '../../shared/aws-clients';
-import {getS3Client} from '../../shared/aws-clients';
 import type {RenderMetadata} from '../../shared/constants';
 import {getExpectedOutName} from './expected-out-name';
 import {getOutputUrlFromMetadata} from './get-output-url-from-metadata';
+import {lambdaHeadCommand} from './io';
 
 export type OutputFileMetadata = {
 	url: string;
@@ -35,12 +34,11 @@ export const findOutputFileInBucket = async ({
 	);
 
 	try {
-		const head = await getS3Client(region, customCredentials).send(
-			new HeadObjectCommand({
-				Bucket: renderBucketName,
-				Key: key,
-			})
-		);
+		const head = await lambdaHeadCommand({
+			bucketName,
+			key,
+			region,
+		});
 		return {
 			lastModified: head.LastModified?.getTime() as number,
 			size: head.ContentLength as number,

packages/lambda/src/functions/helpers/io.ts

@@ -2,6 +2,7 @@ import type {_Object} from '@aws-sdk/client-s3';
 import {
 	DeleteObjectCommand,
 	GetObjectCommand,
+	HeadObjectCommand,
 	ListObjectsV2Command,
 	PutObjectCommand,
 } from '@aws-sdk/client-s3';
@@ -152,3 +153,24 @@ export const lambdaReadFile = async ({
 	);
 	return Body as Readable;
 };
+
+export const lambdaHeadCommand = async ({
+	bucketName,
+	key,
+	region,
+}: {
+	bucketName: string;
+	key: string;
+	region: AwsRegion;
+}): Promise<{
+	LastModified?: Date | undefined;
+	ContentLength?: number | undefined;
+}> => {
+	const head = await getS3Client(region, null).send(
+		new HeadObjectCommand({
+			Bucket: bucketName,
+			Key: key,
+		})
+	);
+	return head;
+};

packages/lambda/src/functions/launch.ts

@@ -38,6 +38,7 @@ import {concatVideosS3} from './helpers/concat-videos';
 import {createPostRenderData} from './helpers/create-post-render-data';
 import {cleanupFiles} from './helpers/delete-chunks';
 import {getExpectedOutName} from './helpers/expected-out-name';
+import {findOutputFileInBucket} from './helpers/find-output-file-in-bucket';
 import {getBrowserInstance} from './helpers/get-browser-instance';
 import {getCurrentRegionInFunction} from './helpers/get-current-region';
 import {getFilesToDelete} from './helpers/get-files-to-delete';
@@ -215,6 +216,7 @@ const innerLaunchHandler = async (params: LambdaPayload, options: Options) => {
 		};
 		return payload;
 	});
+
 	const renderMetadata: RenderMetadata = {
 		startedDate,
 		videoConfig: comp,
@@ -243,6 +245,41 @@ const innerLaunchHandler = async (params: LambdaPayload, options: Options) => {
 		privacy: params.privacy,
 	};
 
+	const {key, renderBucketName, customCredentials} = getExpectedOutName(
+		renderMetadata,
+		params.bucketName,
+		typeof params.outName === 'string' || typeof params.outName === 'undefined'
+			? null
+			: params.outName?.s3OutputProvider ?? null
+	);
+
+	const output = await findOutputFileInBucket({
+		bucketName: params.bucketName,
+		customCredentials,
+		region: getCurrentRegionInFunction(),
+		renderMetadata,
+	});
+
+	if (output) {
+		if (params.overwrite) {
+			console.info(
+				'Deleting',
+				{bucketName: renderBucketName, key},
+				'because it already existed and will be overwritten'
+			);
+			await lambdaDeleteFile({
+				bucketName: renderBucketName,
+				customCredentials,
+				key,
+				region: getCurrentRegionInFunction(),
+			});
+		} else {
+			throw new TypeError(
+				`Output file "${key}" in bucket "${renderBucketName}" in region "${getCurrentRegionInFunction()}" already exists. Delete it before re-rendering, or use the overwrite option to delete it before render."`
+			);
+		}
+	}
+
 	await lambdaWriteFile({
 		bucketName: params.bucketName,
 		key: renderMetadataKey(params.renderId),
@@ -339,14 +376,6 @@ const innerLaunchHandler = async (params: LambdaPayload, options: Options) => {
 		encodingStop = Date.now();
 	}
 
-	const {key, renderBucketName, customCredentials} = getExpectedOutName(
-		renderMetadata,
-		params.bucketName,
-		typeof params.outName === 'string' || typeof params.outName === 'undefined'
-			? null
-			: params.outName?.s3OutputProvider ?? null
-	);
-
 	const outputSize = fs.statSync(outfile);
 
 	await lambdaWriteFile({

packages/lambda/src/functions/start.ts

@@ -74,6 +74,7 @@ export const startHandler = async (params: LambdaPayload, options: Options) => {
 		concurrencyPerLambda: params.concurrencyPerLambda,
 		downloadBehavior: params.downloadBehavior,
 		muted: params.muted,
+		overwrite: params.overwrite,
 	};
 	await getLambdaClient(getCurrentRegionInFunction()).send(
 		new InvokeCommand({

packages/lambda/src/shared/constants.ts

@@ -225,6 +225,7 @@ export type LambdaPayloads = {
 		downloadBehavior: DownloadBehavior;
 		muted: boolean;
 		version: string;
+		overwrite: boolean;
 	};
 	launch: {
 		type: LambdaRoutines.launch;
@@ -254,6 +255,7 @@ export type LambdaPayloads = {
 		concurrencyPerLambda: number;
 		downloadBehavior: DownloadBehavior;
 		muted: boolean;
+		overwrite: boolean;
 	};
 	status: {
 		type: LambdaRoutines.status;

packages/lambda/src/test/integration/renders/gif.test.ts

@@ -58,6 +58,7 @@ test('Should make a distributed GIF', async () => {
 			downloadBehavior: {type: 'play-in-browser'},
 			muted: false,
 			version: VERSION,
+			overwrite: true,
 		},
 		extraContext
 	);

packages/lambda/src/test/integration/renders/old-version.test.ts

@@ -57,6 +57,7 @@ test('Should fail when using an incompatible version', async () => {
 			},
 			muted: false,
 			version: VERSION,
+			overwrite: true,
 		},
 		extraContext
 	);

packages/lambda/src/test/integration/renders/other-bucket.test.ts

@@ -61,6 +61,7 @@ test('Should be able to render to another bucket', async () => {
 			},
 			muted: false,
 			version: VERSION,
+			overwrite: true,
 		},
 		extraContext
 	);

packages/lambda/src/test/integration/renders/silent-audio.test.ts

@@ -58,6 +58,7 @@ test('Should add silent audio if there is no audio', async () => {
 			},
 			muted: false,
 			version: VERSION,
+			overwrite: true,
 		},
 		extraContext
 	);

packages/lambda/src/test/integration/renders/transparent-webm.test.ts

@@ -61,6 +61,7 @@ test('Should make a transparent video', async () => {
 			},
 			muted: false,
 			version: VERSION,
+			overwrite: true,
 		},
 		extraContext
 	);

packages/renderer/src/index.ts

@@ -37,7 +37,6 @@ import {isEqualOrBelowLogLevel, isValidLogLevel, logLevels} from './log-level';
 import {mimeContentType, mimeLookup} from './mime-types';
 import {normalizeServeUrl} from './normalize-serve-url';
 import {killAllBrowsers} from './open-browser';
-import {DEFAULT_OVERWRITE} from './overwrite';
 import {parseStack} from './parse-browser-error-stack';
 import * as perf from './perf';
 import {
@@ -149,7 +148,6 @@ export const RenderInternals = {
 	validateSelectedCrfAndCodecCombination,
 	validImageFormats,
 	validCodecs,
-	DEFAULT_OVERWRITE,
 	DEFAULT_PIXEL_FORMAT,
 	validateQuality,
 	validateFrame,