feat: signed upload url (#158)

Co-authored-by: Inian <inian1234@gmail.com>

infra/docker-compose.yml

@@ -12,19 +12,6 @@ services:
     ports:
       - 8000:8000/tcp
       - 8443:8443/tcp
-  rest:
-    image: postgrest/postgrest:latest
-    ports:
-      - '3000:3000'
-    depends_on:
-      storage:
-        condition: service_healthy
-    restart: always
-    environment:
-      PGRST_DB_URI: postgres://postgres:postgres@db:5432/postgres
-      PGRST_DB_SCHEMA: public, storage
-      PGRST_DB_ANON_ROLE: postgres
-      PGRST_JWT_SECRET: super-secret-jwt-token-with-at-least-32-characters-long
   storage:
     build:
       context: ./storage
@@ -51,6 +38,7 @@ services:
       FILE_STORAGE_BACKEND_PATH: /tmp/storage
       ENABLE_IMAGE_TRANSFORMATION: "true"
       IMGPROXY_URL: http://imgproxy:8080
+      DEBUG: "knex:*"
     volumes:
       - assets-volume:/tmp/storage
     healthcheck:

infra/postgres/dummy-data.sql

@@ -38,6 +38,8 @@ INSERT INTO "storage"."objects" ("id", "bucket_id", "name", "owner", "created_at
 -- add policies
 -- allows user to CRUD all buckets
 CREATE POLICY crud_buckets ON storage.buckets for all USING (auth.uid() = '317eadce-631a-4429-a0bb-f19a7a517b4a');
+CREATE POLICY crud_objects ON storage.objects for all USING (auth.uid() = '317eadce-631a-4429-a0bb-f19a7a517b4a');
+
 -- allow public CRUD acccess to the public folder in bucket2
 CREATE POLICY crud_public_folder ON storage.objects for all USING (bucket_id='bucket2' and (storage.foldername(name))[1] = 'public');
 -- allow public CRUD acccess to a particular file in bucket2

infra/storage/Dockerfile

@@ -1,3 +1,3 @@
-FROM supabase/storage-api:v0.29.1
+FROM supabase/storage-api:v0.35.1
 
 RUN apk add curl --no-cache

src/packages/StorageFileApi.ts

@@ -24,6 +24,18 @@ const DEFAULT_FILE_OPTIONS: FileOptions = {
   upsert: false,
 }
 
+type FileBody =
+  | ArrayBuffer
+  | ArrayBufferView
+  | Blob
+  | Buffer
+  | File
+  | FormData
+  | NodeJS.ReadableStream
+  | ReadableStream<Uint8Array>
+  | URLSearchParams
+  | string
+
 export default class StorageFileApi {
   protected url: string
   protected headers: { [key: string]: string }
@@ -52,17 +64,7 @@ export default class StorageFileApi {
   private async uploadOrUpdate(
     method: 'POST' | 'PUT',
     path: string,
-    fileBody:
-      | ArrayBuffer
-      | ArrayBufferView
-      | Blob
-      | Buffer
-      | File
-      | FormData
-      | NodeJS.ReadableStream
-      | ReadableStream<Uint8Array>
-      | URLSearchParams
-      | string,
+    fileBody: FileBody,
     fileOptions?: FileOptions
   ): Promise<
     | {
@@ -101,7 +103,7 @@ export default class StorageFileApi {
         method,
         body: body as BodyInit,
         headers,
-        ...(options?.duplex ? { duplex: options.duplex } : {})
+        ...(options?.duplex ? { duplex: options.duplex } : {}),
       })
 
       if (res.ok) {
@@ -130,17 +132,7 @@ export default class StorageFileApi {
    */
   async upload(
     path: string,
-    fileBody:
-      | ArrayBuffer
-      | ArrayBufferView
-      | Blob
-      | Buffer
-      | File
-      | FormData
-      | NodeJS.ReadableStream
-      | ReadableStream<Uint8Array>
-      | URLSearchParams
-      | string,
+    fileBody: FileBody,
     fileOptions?: FileOptions
   ): Promise<
     | {
@@ -155,6 +147,115 @@ export default class StorageFileApi {
     return this.uploadOrUpdate('POST', path, fileBody, fileOptions)
   }
 
+  /**
+   * Upload a file with a token generated from `createUploadSignedUrl`.
+   * @param path The file path, including the file name. Should be of the format `folder/subfolder/filename.png`. The bucket must already exist before attempting to upload.
+   * @param token The token generated from `createUploadSignedUrl`
+   * @param fileBody The body of the file to be stored in the bucket.
+   */
+  async uploadToSignedUrl(
+    path: string,
+    token: string,
+    fileBody: FileBody,
+    fileOptions?: FileOptions
+  ) {
+    const cleanPath = this._removeEmptyFolders(path)
+    const _path = this._getFinalPath(cleanPath)
+
+    const url = new URL(this.url + `/object/upload/sign/${_path}`)
+    url.searchParams.set('token', token)
+
+    try {
+      let body
+      const options = { upsert: DEFAULT_FILE_OPTIONS.upsert, ...fileOptions }
+      const headers: Record<string, string> = {
+        ...this.headers,
+        ...{ 'x-upsert': String(options.upsert as boolean) },
+      }
+
+      if (typeof Blob !== 'undefined' && fileBody instanceof Blob) {
+        body = new FormData()
+        body.append('cacheControl', options.cacheControl as string)
+        body.append('', fileBody)
+      } else if (typeof FormData !== 'undefined' && fileBody instanceof FormData) {
+        body = fileBody
+        body.append('cacheControl', options.cacheControl as string)
+      } else {
+        body = fileBody
+        headers['cache-control'] = `max-age=${options.cacheControl}`
+        headers['content-type'] = options.contentType as string
+      }
+
+      const res = await this.fetch(url.toString(), {
+        method: 'PUT',
+        body: body as BodyInit,
+        headers,
+      })
+
+      if (res.ok) {
+        return {
+          data: { path: cleanPath },
+          error: null,
+        }
+      } else {
+        const error = await res.json()
+        return { data: null, error }
+      }
+    } catch (error) {
+      if (isStorageError(error)) {
+        return { data: null, error }
+      }
+
+      throw error
+    }
+  }
+
+  /**
+   * Creates a signed upload URL.
+   * Signed upload URLs can be used upload files to the bucket without further authentication.
+   * They are valid for one minute.
+   * @param path The file path, including the current file name. For example `folder/image.png`.
+   */
+  async createSignedUploadUrl(
+    path: string
+  ): Promise<
+    | {
+        data: { signedUrl: string; token: string; path: string }
+        error: null
+      }
+    | {
+        data: null
+        error: StorageError
+      }
+  > {
+    try {
+      let _path = this._getFinalPath(path)
+
+      const data = await post(
+        this.fetch,
+        `${this.url}/object/upload/sign/${_path}`,
+        {},
+        { headers: this.headers }
+      )
+
+      const url = new URL(this.url + data.url)
+
+      const token = url.searchParams.get('token')
+
+      if (!token) {
+        throw new StorageError('No token returned by API')
+      }
+
+      return { data: { signedUrl: url.toString(), path, token }, error: null }
+    } catch (error) {
+      if (isStorageError(error)) {
+        return { data: null, error }
+      }
+
+      throw error
+    }
+  }
+
   /**
    * Replaces an existing file at the specified path with a new one.
    *

test/__snapshots__/storageApi.test.ts.snap

@@ -3,17 +3,17 @@
 exports[`bucket api Get bucket by id 1`] = `
 {
   "allowed_mime_types": null,
-  "created_at": "2021-02-17T04:43:32.770206+00:00",
+  "created_at": "2021-02-17T04:43:32.770Z",
   "file_size_limit": 0,
   "id": "bucket2",
   "name": "bucket2",
   "owner": "4d56e902-f0a0-4662-8448-a4d9e643c142",
   "public": false,
-  "updated_at": "2021-02-17T04:43:32.770206+00:00",
+  "updated_at": "2021-02-17T04:43:32.770Z",
 }
 `;
 
-exports[`bucket api Get bucket with wrong id 1`] = `[StorageApiError: The resource was not found]`;
+exports[`bucket api Get bucket with wrong id 1`] = `[StorageApiError: Bucket not found]`;
 
 exports[`bucket api delete bucket 1`] = `
 {

test/storageApi.test.ts

@@ -3,7 +3,7 @@ import { StorageClient } from '../src/index'
 // TODO: need to setup storage-api server for this test
 const URL = 'http://localhost:8000/storage/v1'
 const KEY =
-  'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJhdWQiOiIiLCJzdWIiOiIzMTdlYWRjZS02MzFhLTQ0MjktYTBiYi1mMTlhN2E1MTdiNGEiLCJSb2xlIjoicG9zdGdyZXMifQ.pZobPtp6gDcX0UbzMmG3FHSlg4m4Q-22tKtGWalOrNo'
+  'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJPbmxpbmUgSldUIEJ1aWxkZXIiLCJpYXQiOjE2ODA5NjcxMTUsImV4cCI6MTcxMjUwMzI1MywiYXVkIjoiIiwic3ViIjoiMzE3ZWFkY2UtNjMxYS00NDI5LWEwYmItZjE5YTdhNTE3YjRhIiwicm9sZSI6ImF1dGhlbnRpY2F0ZWQifQ.NNzc54y9cZ2QLUHVSrCPOcGE2E0i8ouldc-AaWLsI08'
 
 const storage = new StorageClient(URL, { Authorization: `Bearer ${KEY}` })
 const newBucketName = `my-new-bucket-${Date.now()}`

test/storageFileApi.test.ts

@@ -9,7 +9,7 @@ import fetch from 'cross-fetch'
 // TODO: need to setup storage-api server for this test
 const URL = 'http://localhost:8000/storage/v1'
 const KEY =
-  'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJhdWQiOiIiLCJzdWIiOiIzMTdlYWRjZS02MzFhLTQ0MjktYTBiYi1mMTlhN2E1MTdiNGEiLCJSb2xlIjoicG9zdGdyZXMifQ.pZobPtp6gDcX0UbzMmG3FHSlg4m4Q-22tKtGWalOrNo'
+  'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJPbmxpbmUgSldUIEJ1aWxkZXIiLCJpYXQiOjE2ODA5NjcxMTUsImV4cCI6MTcxMjUwMzI1MywiYXVkIjoiIiwic3ViIjoiMzE3ZWFkY2UtNjMxYS00NDI5LWEwYmItZjE5YTdhNTE3YjRhIiwicm9sZSI6ImF1dGhlbnRpY2F0ZWQifQ.NNzc54y9cZ2QLUHVSrCPOcGE2E0i8ouldc-AaWLsI08'
 
 const storage = new StorageClient(URL, { Authorization: `Bearer ${KEY}` })
 
@@ -67,7 +67,9 @@ describe('Object API', () => {
     })
 
     test('sign url', async () => {
-      await storage.from(bucketName).upload(uploadPath, file)
+      const uploadRes = await storage.from(bucketName).upload(uploadPath, file)
+      expect(uploadRes.error).toBeNull()
+
       const res = await storage.from(bucketName).createSignedUrl(uploadPath, 2000)
 
       expect(res.error).toBeNull()
@@ -215,6 +217,52 @@ describe('Object API', () => {
         statusCode: '422',
       })
     })
+
+    test('sign url for upload', async () => {
+      const res = await storage.from(bucketName).createSignedUploadUrl(uploadPath)
+
+      expect(res.error).toBeNull()
+      expect(res.data?.path).toBe(uploadPath)
+      expect(res.data?.token).toBeDefined()
+      expect(res.data?.signedUrl).toContain(`${URL}/object/upload/sign/${bucketName}/${uploadPath}`)
+    })
+
+    test('can upload with a signed url', async () => {
+      const { data, error } = await storage.from(bucketName).createSignedUploadUrl(uploadPath)
+
+      expect(error).toBeNull()
+      assert(data?.path)
+
+      const uploadRes = await storage
+        .from(bucketName)
+        .uploadToSignedUrl(data.path, data.token, file)
+
+      expect(uploadRes.error).toBeNull()
+      expect(uploadRes.data?.path).toEqual(uploadPath)
+    })
+
+    test('cannot upload to a signed url twice', async () => {
+      const { data, error } = await storage.from(bucketName).createSignedUploadUrl(uploadPath)
+
+      expect(error).toBeNull()
+      assert(data?.path)
+
+      const uploadRes = await storage
+        .from(bucketName)
+        .uploadToSignedUrl(data.path, data.token, file)
+
+      expect(uploadRes.error).toBeNull()
+      expect(uploadRes.data?.path).toEqual(uploadPath)
+
+      const uploadRes2 = await storage
+        .from(bucketName)
+        .uploadToSignedUrl(data.path, data.token, file)
+      expect(uploadRes2.error).toEqual({
+        error: 'Duplicate',
+        message: 'The resource already exists',
+        statusCode: '409',
+      })
+    })
   })
 
   describe('File operations', () => {