Skip to content

Commit

Permalink
Add createMetadata method and refactor finalizeUpload to take upload,…
Browse files Browse the repository at this point in the history 
… not id
  • Loading branch information
@tohhsinpei
tohhsinpei committed Mar 1, 2022
1 parent ff4112f commit 407a762
Show file tree
Hide file tree
Showing 5 changed files with 194 additions and 86 deletions.
124 changes: 116 additions & 8 deletions scripts/storage-emulator-integration/tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1340,9 +1340,14 @@ describe("Storage emulator", () => {
"X-Goog-Upload-Command": "upload, finalize",
})
.expect(200);

await supertest(STORAGE_EMULATOR_HOST)
.get(`/v0/b/${storageBucket}/o/test_upload.jpg`)
.set({ Authorization: "Bearer owner" })
.expect(200);
});

it("#uploadResumableHandlesAuthError", async () => {
it("should return 403 when resumable upload is unauthenticated", async () => {
const uploadURL = await supertest(STORAGE_EMULATOR_HOST)
.post(
`/v0/b/${storageBucket}/o/test_upload.jpg?uploadType=resumable&name=test_upload.jpg`
Expand All @@ -1362,14 +1367,117 @@ describe("Storage emulator", () => {
"X-Goog-Upload-Command": "upload, finalize",
})
.expect(403);
});

await supertest(STORAGE_EMULATOR_HOST)
.put(uploadURL.pathname + uploadURL.search)
.set({
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "cancel",
})
.expect(200);
describe("cancels upload", () => {
it("should cancel upload successfully", async () => {
const uploadURL = await supertest(STORAGE_EMULATOR_HOST)
.post(
`/v0/b/${storageBucket}/o/test_upload.jpg?uploadType=resumable&name=test_upload.jpg`
)
.set({
Authorization: "Bearer owner",
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "start",
})
.expect(200)
.then((res) => new URL(res.header["x-goog-upload-url"]));

await supertest(STORAGE_EMULATOR_HOST)
.put(uploadURL.pathname + uploadURL.search)
.set({
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "cancel",
})
.expect(200);

await supertest(STORAGE_EMULATOR_HOST)
.get(`/v0/b/${storageBucket}/o/test_upload.jpg`)
.set({ Authorization: "Bearer owner" })
.expect(404);
});

it("should return 200 when cancelling already cancelled upload", async () => {
const uploadURL = await supertest(STORAGE_EMULATOR_HOST)
.post(
`/v0/b/${storageBucket}/o/test_upload.jpg?uploadType=resumable&name=test_upload.jpg`
)
.set({
Authorization: "Bearer owner",
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "start",
})
.expect(200)
.then((res) => new URL(res.header["x-goog-upload-url"]));

await supertest(STORAGE_EMULATOR_HOST)
.put(uploadURL.pathname + uploadURL.search)
.set({
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "cancel",
})
.expect(200);

await supertest(STORAGE_EMULATOR_HOST)
.put(uploadURL.pathname + uploadURL.search)
.set({
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "cancel",
})
.expect(200);
});

it("should return 400 when cancelling finalized resumable upload", async () => {
const uploadURL = await supertest(STORAGE_EMULATOR_HOST)
.post(
`/v0/b/${storageBucket}/o/test_upload.jpg?uploadType=resumable&name=test_upload.jpg`
)
.set({
Authorization: "Bearer owner",
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "start",
})
.expect(200)
.then((res) => new URL(res.header["x-goog-upload-url"]));

await supertest(STORAGE_EMULATOR_HOST)
.put(uploadURL.pathname + uploadURL.search)
.set({
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "upload, finalize",
})
.expect(200);

await supertest(STORAGE_EMULATOR_HOST)
.put(uploadURL.pathname + uploadURL.search)
.set({
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "cancel",
})
.expect(400);
});

it("should return 404 when cancelling non-existent upload", async () => {
const uploadURL = await supertest(STORAGE_EMULATOR_HOST)
.post(
`/v0/b/${storageBucket}/o/test_upload.jpg?uploadType=resumable&name=test_upload.jpg`
)
.set({
Authorization: "Bearer owner",
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "start",
})
.expect(200)
.then((res) => new URL(res.header["x-goog-upload-url"]));

await supertest(STORAGE_EMULATOR_HOST)
.put(uploadURL.pathname + uploadURL.search.replace(/(upload_id=).*?(&)/, "$1foo$2"))
.set({
"X-Goog-Upload-Protocol": "resumable",
"X-Goog-Upload-Command": "cancel",
})
.expect(404);
});
});
});

Expand Down
24 changes: 12 additions & 12 deletions src/emulator/storage/apis/firebase.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -493,12 +493,13 @@ export function createFirebaseEndpoints(emulator: StorageEmulator): Router {
}

if (uploadCommand == "cancel") {
const upload = storageLayer.cancelUpload(uploadId);
if (!upload) {
res.sendStatus(400);
return;
const upload = storageLayer.queryUpload(uploadId);
if (upload) {
const cancelled = storageLayer.cancelUpload(upload);
res.sendStatus(cancelled ? 200 : 400);
} else {
res.sendStatus(404);
}
res.sendStatus(200);
return;
}

Expand Down Expand Up @@ -530,12 +531,11 @@ export function createFirebaseEndpoints(emulator: StorageEmulator): Router {
}

if (uploadCommand.includes("finalize")) {
const finalizedUpload = storageLayer.finishUpload(uploadId);
if (!finalizedUpload) {
upload = storageLayer.queryUpload(uploadId);
if (!upload) {
res.sendStatus(400);
return;
}
upload = finalizedUpload.upload;

// For resumable uploads, we check auth on finalization in case of byte-dependant rules
if (
Expand All @@ -546,7 +546,7 @@ export function createFirebaseEndpoints(emulator: StorageEmulator): Router {
path: operationPath,
authorization: upload.authorization,
file: {
after: storageLayer.getMetadata(req.params.bucketId, name)?.asRulesResource(),
after: storageLayer.createMetadata(upload).asRulesResource(),
},
}))
) {
Expand All @@ -560,14 +560,14 @@ export function createFirebaseEndpoints(emulator: StorageEmulator): Router {
}

res.header("x-goog-upload-status", "final");
storageLayer.persistUpload(finalizedUpload);
const uploadedFile = storageLayer.finalizeUpload(upload);

const md = finalizedUpload.file.metadata;
const md = uploadedFile.metadata;
if (md.downloadTokens.length == 0) {
md.addDownloadToken();
}

res.json(new OutgoingFirebaseMetadata(finalizedUpload.file.metadata));
res.json(new OutgoingFirebaseMetadata(uploadedFile.metadata));
} else if (!upload) {
res.sendStatus(400);
return;
Expand Down
12 changes: 3 additions & 9 deletions src/emulator/storage/apis/gcloud.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,20 +123,14 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
});
});

let upload = storageLayer.uploadBytes(uploadId, req.body);

const upload = storageLayer.uploadBytes(uploadId, req.body);
if (!upload) {
res.sendStatus(400);
return;
}

const finalizedUpload = storageLayer.finalizeUpload(uploadId);
if (!finalizedUpload) {
res.sendStatus(400);
return;
}
upload = finalizedUpload.upload;
res.status(200).json(new CloudStorageObjectMetadata(finalizedUpload.file.metadata)).send();
const uploadedFile = storageLayer.finalizeUpload(upload);
res.status(200).json(new CloudStorageObjectMetadata(uploadedFile.metadata)).send();
});

gcloudStorageAPI.post("/b/:bucketId/o/:objectId/acl", (req, res) => {
Expand Down
94 changes: 42 additions & 52 deletions src/emulator/storage/files.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,6 @@ export enum UploadStatus {
FINISHED,
}

export type FinalizedUpload = {
upload: ResumableUpload;
file: StoredFile;
};

export class StorageLayer {
private _files!: Map<string, StoredFile>;
private _uploads!: Map<string, ResumableUpload>;
Expand Down Expand Up @@ -173,6 +168,27 @@ export class StorageLayer {
return;
}

/**
* Generates metadata for an uploaded file. Generally, this should only be used for finalized
* uploads, unless needed for security rule checks.
* @param upload The upload corresponding to the file for which to generate metadata.
* @returns Metadata for uploaded file.
*/
public createMetadata(upload: ResumableUpload): StoredFileMetadata {
const bytes = this._persistence.readBytes(upload.fileLocation, upload.currentBytesUploaded);
return new StoredFileMetadata(
{
name: upload.objectId,
bucket: upload.bucketId,
contentType: "",
contentEncoding: upload.metadata.contentEncoding,
customMetadata: upload.metadata.metadata,
},
this._cloudFunctions,
bytes
);
}

public getBytes(
bucket: string,
object: string,
Expand Down Expand Up @@ -216,17 +232,18 @@ export class StorageLayer {
return this._uploads.get(uploadId);
}

public cancelUpload(uploadId: string): ResumableUpload | undefined {
const upload = this._uploads.get(uploadId);
if (!upload) {
return undefined;
}

if (upload.status !== UploadStatus.FINISHED) {
/**
* Deletes partially uploaded file from persistence layer and updates its status. Cancelling
* an upload is idempotent.
* @param upload The upload to be cancelled.
* @returns Whether the upload was cancelled (i.e. its initial status was ACTIVE or CANCELLED).
*/
public cancelUpload(upload: ResumableUpload): boolean {
if (upload.status === UploadStatus.ACTIVE) {
this._persistence.deleteFile(upload.fileLocation);
upload.status = UploadStatus.CANCELLED;
}
upload.status = UploadStatus.CANCELLED;
return upload;
return upload.status === UploadStatus.CANCELLED;
}

public uploadBytes(uploadId: string, bytes: Buffer): ResumableUpload | undefined {
Expand Down Expand Up @@ -270,53 +287,26 @@ export class StorageLayer {
return this._persistence.deleteAll();
}

public finishUpload(uploadId: string): FinalizedUpload | undefined {
const upload = this._uploads.get(uploadId);

if (!upload) {
return undefined;
}

/**
* Stores the uploaded file with generated metadata and triggers Object Finalize Cloud Functions.
* @param upload The upload to finalize.
* @returns The stored file.
*/
public finalizeUpload(upload: ResumableUpload): StoredFile {
upload.status = UploadStatus.FINISHED;

const metadata = this.createMetadata(upload);
const filePath = this.path(upload.bucketId, upload.objectId);
const file = new StoredFile(metadata, filePath);

const bytes = this._persistence.readBytes(upload.fileLocation, upload.currentBytesUploaded);
const finalMetadata = new StoredFileMetadata(
{
name: upload.objectId,
bucket: upload.bucketId,
contentType: "",
contentEncoding: upload.metadata.contentEncoding,
customMetadata: upload.metadata.metadata,
},
this._cloudFunctions,
bytes
);
const file = new StoredFile(finalMetadata, filePath);
this._files.set(filePath, file);

return {upload, file};
}

public persistUpload(finalizedUpload: FinalizedUpload): void {
const {upload, file} = finalizedUpload;
const filePath = this.path(upload.bucketId, upload.objectId);

this._persistence.deleteFile(filePath, true);
this._persistence.deleteFile(filePath, /* failSilently = */ true);
this._persistence.renameFile(upload.fileLocation, filePath);

this._cloudFunctions.dispatch("finalize", new CloudStorageObjectMetadata(file.metadata));
}

public finalizeUpload(uploadId: string): FinalizedUpload | undefined {
const finalizedUpload = this.finishUpload(uploadId);

if (!finalizedUpload) {
return undefined;
}

this.persistUpload(finalizedUpload);
return finalizedUpload;
return file;
}

public oneShotUpload(
Expand Down
26 changes: 21 additions & 5 deletions src/test/emulators/storage/files.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,29 @@ describe("files", () => {
expect(deserialized).to.deep.equal(metadata);
});

it("finishUpload persists file to memory", () => {
it("should store file in memory when upload is finalized", () => {
const storageLayer = new StorageLayer("project");
const { uploadId } = storageLayer.startUpload("bucket", "object", "mime/type", {
const bytesToWrite = "Hello, World!";

const upload = storageLayer.startUpload("bucket", "object", "mime/type", {
contentType: "mime/type",
});
storageLayer.uploadBytes(upload.uploadId, Buffer.from(bytesToWrite));
storageLayer.finalizeUpload(upload);

expect(storageLayer.getBytes("bucket", "object")?.includes(bytesToWrite));
expect(storageLayer.getMetadata("bucket", "object")?.size).equals(bytesToWrite.length);
});

it("should delete file from persistence layer when upload is cancelled", () => {
const storageLayer = new StorageLayer("project");

const upload = storageLayer.startUpload("bucket", "object", "mime/type", {
contentType: "mime/type",
});
storageLayer.uploadBytes(uploadId, Buffer.alloc(0));
storageLayer.finishUpload(uploadId);
expect(storageLayer.getMetadata("bucket", "object")).not.to.equal(undefined);
storageLayer.uploadBytes(upload.uploadId, Buffer.alloc(0));
storageLayer.cancelUpload(upload);

expect(storageLayer.getMetadata("bucket", "object")).to.equal(undefined);
});
});

0 comments on commit 407a762

Please sign in to comment.