Fix issue with importing Storage Emulator data with nested directory structure

CHANGELOG.md

@@ -2,3 +2,4 @@
 - Fixes bug where GCS metadataUpdate cloud functions were triggered in incorrect situations (#3398).
 - Fixes bug where quoted escape sequences in .env files were incompletely unescaped. (#4270)
 - Fixes Storage Emulator ruleset file watcher (#4337).
+- Fixes issue with importing Storage Emulator data exported prior to v10.3.0 (#4358).

scripts/storage-emulator-integration/import/flattened-emulator-data/firebase-export-metadata.json

@@ -0,0 +1,7 @@
+{
+  "version": "10.4.2",
+  "storage": {
+    "version": "10.4.2",
+    "path": "storage_export"
+  }
+}

scripts/storage-emulator-integration/import/flattened-emulator-data/storage_export/buckets.json

@@ -0,0 +1,7 @@
+{
+  "buckets": [
+    {
+      "id": "fake-project-id.appspot.com"
+    }
+  ]
+}

scripts/storage-emulator-integration/import/flattened-emulator-data/storage_export/metadata/fake-project-id.appspot.com%2Ftest_upload.jpg.json

@@ -0,0 +1,20 @@
+{
+  "name": "test_upload.jpg",
+  "bucket": "fake-project-id.appspot.com",
+  "contentType": "application/octet-stream",
+  "metageneration": 1,
+  "generation": 1648084940926,
+  "storageClass": "STANDARD",
+  "contentDisposition": "inline",
+  "cacheControl": "public, max-age=3600",
+  "contentEncoding": "identity",
+  "downloadTokens": [
+    "c3c71086-95a8-445d-96e7-f625972de4b0"
+  ],
+  "etag": "PQJQBXRweACX9yRsBEInQjOJ/0s",
+  "timeCreated": "2022-03-24T01:22:20.926Z",
+  "updated": "2022-03-24T01:22:20.926Z",
+  "size": 0,
+  "md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==",
+  "crc32c": "0"
+}

scripts/storage-emulator-integration/import/nested-emulator-data/firebase-export-metadata.json

@@ -0,0 +1,7 @@
+{
+  "version": "10.4.2",
+  "storage": {
+    "version": "10.4.2",
+    "path": "storage_export"
+  }
+}

scripts/storage-emulator-integration/import/nested-emulator-data/storage_export/buckets.json

@@ -0,0 +1,7 @@
+{
+  "buckets": [
+    {
+      "id": "fake-project-id.appspot.com"
+    }
+  ]
+}

scripts/storage-emulator-integration/import/nested-emulator-data/storage_export/metadata/fake-project-id.appspot.com/test_upload.jpg.json

@@ -0,0 +1,20 @@
+{
+  "name": "test_upload.jpg",
+  "bucket": "fake-project-id.appspot.com",
+  "contentType": "application/octet-stream",
+  "metageneration": 1,
+  "generation": 1648084940926,
+  "storageClass": "STANDARD",
+  "contentDisposition": "inline",
+  "cacheControl": "public, max-age=3600",
+  "contentEncoding": "identity",
+  "downloadTokens": [
+    "c3c71086-95a8-445d-96e7-f625972de4b0"
+  ],
+  "etag": "PQJQBXRweACX9yRsBEInQjOJ/0s",
+  "timeCreated": "2022-03-24T01:22:20.926Z",
+  "updated": "2022-03-24T01:22:20.926Z",
+  "size": 0,
+  "md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==",
+  "crc32c": "0"
+}

scripts/storage-emulator-integration/import/tests.ts

@@ -0,0 +1,90 @@
+import * as path from "path";
+import supertest = require("supertest");
+
+import { createTmpDir } from "../../../src/test/emulators/fixtures";
+import { Emulators } from "../../../src/emulator/types";
+import { TriggerEndToEndTest } from "../../integration-helpers/framework";
+import {
+  EMULATORS_SHUTDOWN_DELAY_MS,
+  FIREBASE_EMULATOR_CONFIG,
+  getStorageEmulatorHost,
+  readEmulatorConfig,
+  TEST_SETUP_TIMEOUT,
+} from "../utils";
+
+describe("Import Emulator Data", () => {
+  const FIREBASE_PROJECT = "fake-project-id";
+  const BUCKET = `${FIREBASE_PROJECT}.appspot.com`;
+  const EMULATOR_CONFIG = readEmulatorConfig(FIREBASE_EMULATOR_CONFIG);
+  const STORAGE_EMULATOR_HOST = getStorageEmulatorHost(EMULATOR_CONFIG);
+  const test = new TriggerEndToEndTest(FIREBASE_PROJECT, __dirname, EMULATOR_CONFIG);
+
+  it("retrieves file from imported flattened emulator data", async function (this) {
+    this.timeout(TEST_SETUP_TIMEOUT);
+    await test.startEmulators([
+      "--only",
+      Emulators.STORAGE,
+      "--import",
+      path.join(__dirname, "flattened-emulator-data"),
+      "--export-on-exit",
+      path.join(__dirname, "other-emulator-data"),
+    ]);
+
+    await supertest(STORAGE_EMULATOR_HOST)
+      .get(`/v0/b/${BUCKET}/o/test_upload.jpg`)
+      .set({ Authorization: "Bearer owner" })
+      .expect(200);
+  });
+
+  it("retrieves file from imported nested emulator data", async function (this) {
+    this.timeout(TEST_SETUP_TIMEOUT);
+    await test.startEmulators([
+      "--only",
+      Emulators.STORAGE,
+      "--import",
+      path.join(__dirname, "flattened-emulator-data"),
+    ]);
+
+    await supertest(STORAGE_EMULATOR_HOST)
+      .get(`/v0/b/${BUCKET}/o/test_upload.jpg`)
+      .set({ Authorization: "Bearer owner" })
+      .expect(200);
+  });
+
+  it("retrieves file from importing previously exported emulator data", async function (this) {
+    this.timeout(TEST_SETUP_TIMEOUT);
+    const tmpDir = createTmpDir("exported-emulator-data");
+
+    // Upload file to Storage and export emulator data to tmp directory
+    await test.startEmulators(["--only", Emulators.STORAGE, "--export-on-exit", tmpDir]);
+    const uploadURL = await supertest(STORAGE_EMULATOR_HOST)
+      .post(`/v0/b/${BUCKET}/o/test_upload.jpg?uploadType=resumable&name=test_upload.jpg`)
+      .set({
+        Authorization: "Bearer owner",
+        "X-Goog-Upload-Protocol": "resumable",
+        "X-Goog-Upload-Command": "start",
+      })
+      .then((res) => new URL(res.header["x-goog-upload-url"]));
+
+    await supertest(STORAGE_EMULATOR_HOST)
+      .put(uploadURL.pathname + uploadURL.search)
+      .set({
+        "X-Goog-Upload-Protocol": "resumable",
+        "X-Goog-Upload-Command": "upload, finalize",
+      });
+
+    await test.stopEmulators();
+
+    // Import emulator data from tmp directory and retrieve file from Storage
+    await test.startEmulators(["--only", Emulators.STORAGE, "--import", tmpDir]);
+    await supertest(STORAGE_EMULATOR_HOST)
+      .get(`/v0/b/${BUCKET}/o/test_upload.jpg`)
+      .set({ Authorization: "Bearer owner" })
+      .expect(200);
+  });
+
+  afterEach(async function (this) {
+    this.timeout(EMULATORS_SHUTDOWN_DELAY_MS);
+    await test.stopEmulators();
+  });
+});

scripts/storage-emulator-integration/run.sh

@@ -9,6 +9,8 @@ firebase setup:emulators:storage
 
 mocha scripts/storage-emulator-integration/rules/*.test.ts
 
+mocha scripts/storage-emulator-integration/import/tests.ts
+
 mocha scripts/storage-emulator-integration/multiple-targets/tests.ts
 
 mocha scripts/storage-emulator-integration/tests.ts

src/emulator/storage/files.ts

@@ -583,12 +583,18 @@ export class StorageLayer {
         continue;
       }
 
-      const file = new StoredFile(metadata, blobPath);
-      this._files.set(blobPath, file);
-    }
+      const decodedBlobPath = decodeURIComponent(blobPath);
+      const decodedMetadataPath = decodeURIComponent(metadataRelPath);
+
+      const blobDiskPath = this._persistence.getDiskPath(decodedBlobPath);
+      const metadataDiskPath = this._persistence.getDiskPath(decodedMetadataPath);
 
-    // Recursively copy all blobs
-    fse.copySync(blobsDir, this.dirPath);
+      const file = new StoredFile(metadata, blobDiskPath);
+      this._files.set(decodedBlobPath, file);
+
+      fse.copyFileSync(f, metadataDiskPath);
+      fse.copyFileSync(blobAbsPath, blobDiskPath);
+    }
   }
 
   private *walkDirSync(dir: string): Generator<string> {