Merge pull request #1332 from remotion-dev/custom-s3-credentials

packages/docs/docs/lambda/custom-destination.md

@@ -0,0 +1,103 @@
+---
+id: custom-destination
+sidebar_label: Custom output destination
+title: Customizing Lambda output destination
+---
+
+By default a render artifact is saved into the same S3 bucket as where the site is located under the key `renders/${renderId}/out.{extension}` (for example: `renders/hy0k2siao8/out.mp4`)
+
+You can modify the output destination by passing a different filename, writing it into a different bucket or even upload it to a different S3-compatible provider.
+
+## Customizing the output name
+
+To customize the output filename, pass `outName: "my-filename.mp4"` to [`renderMediaOnLambda()`](/docs/lambda/rendermediaonlambda#outname) or [`renderStillOnLambda()`](/docs/lambda/renderstillonlambda#outname).
+
+On the CLI, use the [`--out-name`](/docs/lambda/cli/render#--out-name) flag.
+
+The output name must match `/^([0-9a-zA-Z-!_.*'()/]+)$/g`.
+
+## Customizing the output bucket
+
+To render into a different bucket, specify the `outName` option to [`renderMediaOnLambda()`](/docs/lambda/rendermediaonlambda) or [`renderStillOnLambda()`](/docs/lambda/renderstillonlambda) and pass an object with the `key` and `bucketName` values:
+
+```tsx twoslash {13-16}
+// @module: esnext
+// @target: es2017
+import { renderMediaOnLambda } from "@remotion/lambda";
+// ---cut---
+
+const { bucketName, renderId } = await renderMediaOnLambda({
+  region: "us-east-1",
+  functionName: "remotion-render-bds9aab",
+  composition: "MyVideo",
+  serveUrl:
+    "https://remotionlambda-qg35eyp1s1.s3.eu-central-1.amazonaws.com/sites/bf2jrbfkw",
+  inputProps: {},
+  codec: "h264",
+  imageFormat: "jpeg",
+  maxRetries: 1,
+  privacy: "public",
+  outName: {
+    key: "my-output",
+    bucketName: "output-bucket",
+  },
+});
+```
+
+If you like to use this feature:
+
+- You must extend the [default Remotion policy](/docs/lambda/permissions) to allow read and write access to that bucket.
+- The bucket must be in the same region.
+- When calling APIs such as [`downloadMedia()`](/docs/lambda/downloadmedia) or [`getRenderProgress()`](/docs/lambda/getrenderprogress), you must pass the `bucketName` where the site resides in, not the bucket where the video gets saved.
+- The `key` must match `/^([0-9a-zA-Z-!_.*'()]+)$/g`
+- The bucketName must match `/^(?=^.{3,63}$)(?!^(\d+\.)+\d+$)(^(([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])\.)*([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])$)/`.
+
+This feature is not supported from the CLI.
+
+## Saving to another cloud
+
+_Available from v3.2.23_
+
+You can upload the file to another S3-compatible provider. You must pass an `outName` [as specified above](#customizing-the-output-bucket) and also provide an `s3OutputProvider` like in the example below.
+
+```tsx twoslash {13-21}
+// @module: esnext
+// @target: es2017
+import { renderMediaOnLambda } from "@remotion/lambda";
+// ---cut---
+
+const { bucketName, renderId } = await renderMediaOnLambda({
+  region: "us-east-1",
+  functionName: "remotion-render-bds9aab",
+  composition: "MyVideo",
+  serveUrl:
+    "https://remotionlambda-qg35eyp1s1.s3.eu-central-1.amazonaws.com/sites/bf2jrbfkw",
+  inputProps: {},
+  codec: "h264",
+  imageFormat: "jpeg",
+  maxRetries: 1,
+  privacy: "public",
+  outName: {
+    key: "my-output",
+    bucketName: "output-bucket",
+    s3OutputProvider: {
+      endpoint: "https://fra1.digitaloceanspaces.com",
+      accessKeyId: "<DIGITAL_OCEAN_ACCESS_KEY_ID>",
+      secretAccessKey: "<DIGITAL_OCEAN_SECRET_ACCESS_KEY>",
+    },
+  },
+});
+```
+
+In this example, the output file will be uploaded to DigitalOcean Spaces. The cloud provider will give you the endpoint and credentials.
+
+If you want to use this feature, note the following:
+
+- When calling [`downloadMedia()`](/docs/lambda/downloadmedia#bucketname) or [`getRenderProgress()`](/docs/lambda/getrenderprogress#bucketname), you must pass the AWS `bucketName` where the site resides in, not the bucket name of the foreign cloud.
+- When calling [`downloadMedia()`](/docs/lambda/downloadmedia#s3outputprovider) or [`getRenderProgress()`](/docs/lambda/getrenderprogress#s3outputprovider), you must provide the `s3OutputProvider` option with the same credentials again.
+
+This feature is not supported from the CLI.
+
+## See also
+
+- Customizing the filename when a file is downloaded using `downloadBehavior`: For [`renderMediaOnLambda()`](/docs/lambda/rendermediaonlambda#downloadbehavior) and [`renderStillOnLambda()`](/docs/lambda/renderstillonlambda#downloadbehavior)

packages/docs/docs/lambda/downloadmedia.md

@@ -51,12 +51,20 @@ Where the video should be saved. Pass an absolute path, or it will be resolved r
 
 ### `onProgress`
 
+_optional_
+
 Callback function that gets called with the following properties:
 
 - `totalSize` in bytes
 - `downloaded` number of bytes downloaded
 - `percent` relative progress between 0 and 1
 
+### `customCredentials`
+
+_optional, available from v3.2.23_
+
+If the render was saved to a [different cloud](/docs/lambda/custom-destination#saving-to-another-cloud), pass an object with the same `endpoint`, `accessKeyId` and `secretAccessKey` as you passed to [`renderMediaOnLambda()`](/docs/lambda/rendermediaonlambda#outname) or [`renderStillOnLambda()`](/docs/lambda/renderstillonlambda#outname).
+
 ## Return value
 
 Returns a promise resolving to an object with the following properties:

packages/docs/docs/lambda/getawsclient.md

@@ -74,6 +74,20 @@ One of the [supported regions](/docs/lambda/region-selection) of Remotion Lambda
 
 One of `lambda`, `cloudwatch`, `iam`, `servicequotas` and `s3`.
 
+### `customCredentials`
+
+_available from v3.2.23_
+
+Allows you to connect to another cloud provider, useful if you [render your output to a different cloud](/docs/lambda/custom-destination). The value must satisfy the following type:
+
+```ts
+type CustomCredentials = {
+  endpoint: string;
+  accessKeyId: string | null;
+  secretAccessKey: string | null;
+};
+```
+
 ## Return value
 
 An object with two properties:

packages/docs/docs/lambda/getrenderprogress.md

@@ -43,6 +43,12 @@ The region in which the Lambda function is located in.
 
 The name of the function that triggered the render.
 
+### `customCredentials`
+
+_optional, available from v3.2.23_
+
+If the render is going to be saved to a [different cloud](/docs/lambda/custom-destination#saving-to-another-cloud), pass an object with the same `endpoint`, `accessKeyId` and `secretAccessKey` as you passed to [`renderMediaOnLambda()`](/docs/lambda/rendermediaonlambda#outname) or [`renderStillOnLambda()`](/docs/lambda/renderstillonlambda#outname).
+
 ## Response
 
 Returns a promise resolving to an object with the following properties:

packages/docs/docs/lambda/rendermediaonlambda.md

@@ -135,13 +135,9 @@ The file name of the media output.
 
 It can either be:
 
-- `undefined` - it will default to `out` plus the appropriate file extension, for example: `renders/${renderId}/out.mp4`. The outName must match `/^([0-9a-zA-Z-!_.*'()/]+)$/g`.
+- `undefined` - it will default to `out` plus the appropriate file extension, for example: `renders/${renderId}/out.mp4`.
 - A `string` - it will get saved to the same S3 bucket as your site under the key `renders/{renderId}/{outName}`.
-- An object of shape `{ key: string; bucketName: string }`. This will save the render to an arbitrary bucket with an arbitrary key. Note the following restrictions:
-  - You must extend the default Remotion policy to allow read and write access to that bucket.
-  - The bucket must be in the same region.
-  - When calling APIs such as `downloadMedia()` or `getRenderProgress()`, you must pass the bucket name where the site resides in, not the bucket where the video gets saved.
-  - The `key` must match `/^([0-9a-zA-Z-!_.*'()]+)$/g` and the bucketName must match `/^(?=^.{3,63}$)(?!^(\d+\.)+\d+$)(^(([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])\.)*([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])$)/`.
+- An object if you want to render to a different bucket or cloud provider - [see here for detailed instructions](/docs/lambda/custom-destination)
 
 ### `timeoutInMilliseconds?`
 

packages/docs/docs/lambda/renderstillonlambda.md

@@ -108,17 +108,11 @@ Scales the output dimensions by a factor. See [Scaling](/docs/scaling) to learn
 
 _optional_
 
-The file name of the media output.
-
 It can either be:
 
-- `undefined` - it will default to `out` plus the appropriate file extension, for example: `renders/${renderId}/out.mp4`. The outName must match `/^([0-9a-zA-Z-!_.*'()/]+)$/g`.
+- `undefined` - it will default to `out` plus the appropriate file extension, for example: `renders/${renderId}/out.mp4`.
 - A `string` - it will get saved to the same S3 bucket as your site under the key `renders/{renderId}/{outName}`.
-- An object of shape `{ key: string; bucketName: string }`. This will save the render to an arbitrary bucket with an arbitrary key. Note the following restrictions:
-  - You must extend the default Remotion policy to allow read and write access to that bucket.
-  - The bucket must be in the same region.
-  - When calling APIs such as `downloadMedia()` or `getRenderProgress()`, you must pass the bucket name where the site resides in, not the bucket where the video gets saved.
-  - The `key` must match `/^([0-9a-zA-Z-!_.*'()]+)$/g` and the bucketName must match `/^(?=^.{3,63}$)(?!^(\d+\.)+\d+$)(^(([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])\.)*([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])$)/`.
+- An object if you want to render to a different bucket or cloud provider - [see here for detailed instructions](/docs/lambda/custom-destination)
 
 ### `timeoutInMilliseconds?`
 

packages/docs/sidebars.js

@@ -310,6 +310,7 @@ module.exports = {
         "lambda/faq",
         "lambda/light-client",
         "lambda/custom-layers",
+        "lambda/custom-destination",
         "lambda/checklist",
         {
           type: "category",

packages/lambda/src/api/bucket-exists.ts

@@ -12,7 +12,7 @@ export const bucketExistsInRegion = async ({
 	expectedBucketOwner: string | null;
 }) => {
 	try {
-		const bucket = await getS3Client(region).send(
+		const bucket = await getS3Client(region, null).send(
 			new GetBucketLocationCommand({
 				Bucket: bucketName,
 				ExpectedBucketOwner: expectedBucketOwner ?? undefined,

packages/lambda/src/api/clean-items.ts

@@ -25,7 +25,7 @@ export const cleanItems = async ({
 					bucketName: bucket,
 					itemName: object,
 				});
-				await getS3Client(region).send(
+				await getS3Client(region, null).send(
 					new DeleteObjectCommand({
 						Bucket: bucket,
 						Key: object,

packages/lambda/src/api/create-bucket.ts

@@ -9,7 +9,7 @@ export const createBucket = async ({
 	region: AwsRegion;
 	bucketName: string;
 }) => {
-	await getS3Client(region).send(
+	await getS3Client(region, null).send(
 		new CreateBucketCommand({
 			Bucket: bucketName,
 			ACL: 'public-read',

packages/lambda/src/api/download-media.ts

@@ -5,6 +5,7 @@ import {getRenderMetadata} from '../functions/helpers/get-render-metadata';
 import type {LambdaReadFileProgress} from '../functions/helpers/read-with-progress';
 import {lambdaDownloadFileWithProgress} from '../functions/helpers/read-with-progress';
 import type {AwsRegion} from '../pricing/aws-regions';
+import type {CustomCredentials} from '../shared/aws-clients';
 import {getAccountId} from '../shared/get-account-id';
 
 export type DownloadMediaInput = {
@@ -13,13 +14,26 @@ export type DownloadMediaInput = {
 	renderId: string;
 	outPath: string;
 	onProgress?: LambdaReadFileProgress;
+	customCredentials?: CustomCredentials;
 };
 
 export type DownloadMediaOutput = {
 	outputPath: string;
 	sizeInBytes: number;
 };
 
+/**
+ * @description Triggers a render on a lambda given a composition and a lambda function.
+ * @link https://remotion.dev/docs/lambda/downloadmedia
+ * @param params.region The AWS region in which the media resides.
+ * @param params.bucketName The `bucketName` that was specified during the render.
+ * @param params.renderId The `renderId` that was obtainer after triggering the render.
+ * @param params.outPath Where to save the media.
+ * @param params.onProgress Progress callback function - see docs for details.
+ * @param params.customCredentials If the file was saved to a foreign cloud, pass credentials for reading from it.
+ * @returns {Promise<RenderMediaOnLambdaOutput>} See documentation for detailed structure
+ */
+
 export const downloadMedia = async (
 	input: DownloadMediaInput
 ): Promise<DownloadMediaOutput> => {
@@ -35,17 +49,21 @@ export const downloadMedia = async (
 
 	const outputPath = path.resolve(process.cwd(), input.outPath);
 	RenderInternals.ensureOutputDirectory(outputPath);
-	const {key, renderBucketName} = getExpectedOutName(
+
+	const {key, renderBucketName, customCredentials} = getExpectedOutName(
 		renderMetadata,
-		input.bucketName
+		input.bucketName,
+		input.customCredentials ?? null
 	);
+
 	const {sizeInBytes} = await lambdaDownloadFileWithProgress({
 		bucketName: renderBucketName,
 		expectedBucketOwner,
 		key,
 		region: input.region,
 		onProgress: input.onProgress ?? (() => undefined),
 		outputPath,
+		customCredentials,
 	});
 
 	return {

packages/lambda/src/api/enable-s3-website.ts

@@ -9,7 +9,7 @@ export const enableS3Website = async ({
 	region: AwsRegion;
 	bucketName: string;
 }) => {
-	await getS3Client(region).send(
+	await getS3Client(region, null).send(
 		new PutBucketWebsiteCommand({
 			Bucket: bucketName,
 			WebsiteConfiguration: {

packages/lambda/src/api/get-aws-client.ts

@@ -4,12 +4,13 @@ import * as LambdaSDK from '@aws-sdk/client-lambda';
 import * as S3SDK from '@aws-sdk/client-s3';
 import * as ServiceQuotasSDK from '@aws-sdk/client-service-quotas';
 import type {AwsRegion} from '../client';
-import type { ServiceMapping} from '../shared/aws-clients';
+import type {CustomCredentials, ServiceMapping} from '../shared/aws-clients';
 import {getServiceClient} from '../shared/aws-clients';
 
 export type GetAwsClientInput<T extends keyof ServiceMapping> = {
 	region: AwsRegion;
 	service: T;
+	customCredentials?: CustomCredentials | null;
 };
 
 type SdkMapping = {
@@ -30,14 +31,20 @@ export type GetAwsClientOutput<T extends keyof ServiceMapping> = {
  * @link https://remotion.dev/docs/lambda/getawsclient
  * @param {AwsRegion} params.region The region in which the S3 bucket resides in.
  * @param {string} params.service One of `iam`, `s3`, `cloudwatch`, `iam` or `servicequotas`
+ * @param {CustomCredentials} params.customCredentials Pass endpoint and credentials if you want to connect to a different cloud for S3
  * @returns {GetAwsClientOutput<T>} Returns `client` and `sdk` of a AWS service
  */
 export const getAwsClient = <T extends keyof ServiceMapping>({
 	region,
 	service,
+	customCredentials,
 }: GetAwsClientInput<T>): GetAwsClientOutput<T> => {
 	return {
-		client: getServiceClient(region, service),
+		client: getServiceClient({
+			region,
+			service,
+			customCredentials: customCredentials ?? null,
+		}),
 		sdk: {
 			lambda: LambdaSDK,
 			cloudwatch: CloudWatchSDK,

packages/lambda/src/api/get-buckets.ts

@@ -14,7 +14,9 @@ export const getRemotionS3Buckets = async (
 ): Promise<{
 	remotionBuckets: BucketWithLocation[];
 }> => {
-	const {Buckets} = await getS3Client(region).send(new ListBucketsCommand({}));
+	const {Buckets} = await getS3Client(region, null).send(
+		new ListBucketsCommand({})
+	);
 	if (!Buckets) {
 		return {remotionBuckets: []};
 	}
@@ -25,7 +27,7 @@ export const getRemotionS3Buckets = async (
 
 	const locations = await Promise.all(
 		remotionBuckets.map((bucket) => {
-			return getS3Client(region).send(
+			return getS3Client(region, null).send(
 				new GetBucketLocationCommand({
 					Bucket: bucket.Name as string,
 				})

packages/lambda/src/api/get-render-progress.ts

@@ -1,5 +1,6 @@
 import {VERSION} from 'remotion/version';
 import type {AwsRegion} from '../pricing/aws-regions';
+import type {CustomCredentials} from '../shared/aws-clients';
 import {callLambda} from '../shared/call-lambda';
 import type {RenderProgress} from '../shared/constants';
 import {LambdaRoutines} from '../shared/constants';
@@ -9,6 +10,7 @@ export type GetRenderInput = {
 	bucketName: string;
 	renderId: string;
 	region: AwsRegion;
+	s3OutputProvider?: CustomCredentials;
 };
 
 /**
@@ -18,13 +20,15 @@ export type GetRenderInput = {
  * @param {string} params.bucketName The name of the bucket that was used in the render.
  * @param {string} params.renderId The ID of the render that was returned by `renderMediaOnLambda()`.
  * @param {AwsRegion} params.region The region in which the render was triggered.
+ * @param {CustomCredentials} params.s3OutputProvider? Endpoint and credentials if the output file is stored outside of AWS.
  * @returns {Promise<RenderProgress>} See documentation for this function to see all properties on the return object.
  */
 export const getRenderProgress = async ({
 	functionName,
 	bucketName,
 	renderId,
 	region,
+	s3OutputProvider,
 }: GetRenderInput): Promise<RenderProgress> => {
 	const result = await callLambda({
 		functionName,
@@ -33,6 +37,7 @@ export const getRenderProgress = async ({
 			bucketName,
 			renderId,
 			version: VERSION,
+			s3OutputProvider,
 		},
 		region,
 	});