Merge branch 'master' into inlined.functionsv2-warning

.github/workflows/node-test.yml

@@ -25,7 +25,7 @@ jobs:
           cache: npm
           cache-dependency-path: npm-shrinkwrap.json
 
-      - run: npm i -g npm@8
+      - run: npm i -g npm@8.5
       - run: npm ci
       - run: npm run lint:changed-files
 
@@ -45,7 +45,7 @@ jobs:
           cache: npm
           cache-dependency-path: npm-shrinkwrap.json
 
-      - run: npm i -g npm@8
+      - run: npm i -g npm@8.5
       - run: npm ci
       - run: npm test
 
@@ -90,7 +90,7 @@ jobs:
           key: ${{ runner.os }}-firebase-emulators-${{ hashFiles('emulator-cache/**') }}
         continue-on-error: true
 
-      - run: npm i -g npm@8
+      - run: npm i -g npm@8.5
       - run: npm ci
       - run: echo ${{ secrets.service_account_json_base64 }} | base64 -d > ./scripts/service-account.json
       - run: ${{ matrix.script }}
@@ -113,7 +113,7 @@ jobs:
         uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
-      - run: npm i -g npm@8
+      - run: npm i -g npm@8.5
       # --ignore-scripts prevents the `prepare` script from being run.
       - run: npm install --package-lock-only --ignore-scripts
       - run: "git diff --exit-code -- npm-shrinkwrap.json || (echo 'Error: npm-shrinkwrap.json is changed during npm install! Please make sure to use npm >= 8 and commit npm-shrinkwrap.json.' && false)"

CHANGELOG.md

@@ -1 +1,8 @@
-Improve error messages when using alpha features without being in the alpha program (#4354)
+- Improve error messages when using alpha features without being in the alpha program (#4354)
+- Fixes bug where resumable uploads were not setting custom metadata on upload (#3398).
+- Fixes bug where GCS metadataUpdate cloud functions were triggered in incorrect situations (#3398).
+- Fixes bug where quoted escape sequences in .env files were incompletely unescaped. (#4270)
+- Fixes Storage Emulator ruleset file watcher (#4337).
+- Fixes issue with importing Storage Emulator data exported prior to v10.3.0 (#4358).
+- Adds ergonomic improvements to CF3 secret commands to automatically redeploy functions and delete unused secrets (#4130).
+- Fixes issue with alpha users setting timeouts (#4381)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "firebase-tools",
-  "version": "10.4.2",
+  "version": "10.5.0",
   "description": "Command-Line Interface for Firebase",
   "main": "./lib/index.js",
   "bin": {

scripts/emulator-tests/functionsEmulatorRuntime.spec.ts

@@ -2,7 +2,6 @@ import { Change } from "firebase-functions";
 import { DocumentSnapshot } from "firebase-functions/lib/providers/firestore";
 import { expect } from "chai";
 import { IncomingMessage, request } from "http";
-import * as _ from "lodash";
 import * as express from "express";
 import * as fs from "fs";
 import * as sinon from "sinon";
@@ -12,7 +11,7 @@ import { FunctionRuntimeBundles, TIMEOUT_LONG, TIMEOUT_MED, MODULE_ROOT } from "
 import { FunctionsRuntimeBundle, SignatureType } from "../../src/emulator/functionsEmulatorShared";
 import { InvokeRuntimeOpts, FunctionsEmulator } from "../../src/emulator/functionsEmulator";
 import { RuntimeWorker } from "../../src/emulator/functionsRuntimeWorker";
-import { streamToString } from "../../src/utils";
+import { streamToString, cloneDeep } from "../../src/utils";
 import * as registry from "../../src/emulator/registry";
 
 const DO_NOTHING = () => {
@@ -449,7 +448,7 @@ describe("FunctionsEmulator-Runtime", () => {
       }).timeout(TIMEOUT_MED);
 
       it("should return a real databaseURL when RTDB emulator is not running", async () => {
-        const frb = _.cloneDeep(FunctionRuntimeBundles.onRequest);
+        const frb = cloneDeep(FunctionRuntimeBundles.onRequest);
         const worker = await invokeFunction(
           frb,
           () => {

scripts/storage-emulator-integration/import/flattened-emulator-data/firebase-export-metadata.json

@@ -0,0 +1,7 @@
+{
+  "version": "10.4.2",
+  "storage": {
+    "version": "10.4.2",
+    "path": "storage_export"
+  }
+}

scripts/storage-emulator-integration/import/flattened-emulator-data/storage_export/buckets.json

@@ -0,0 +1,7 @@
+{
+  "buckets": [
+    {
+      "id": "fake-project-id.appspot.com"
+    }
+  ]
+}

scripts/storage-emulator-integration/import/flattened-emulator-data/storage_export/metadata/fake-project-id.appspot.com%2Ftest_upload.jpg.json

@@ -0,0 +1,20 @@
+{
+  "name": "test_upload.jpg",
+  "bucket": "fake-project-id.appspot.com",
+  "contentType": "application/octet-stream",
+  "metageneration": 1,
+  "generation": 1648084940926,
+  "storageClass": "STANDARD",
+  "contentDisposition": "inline",
+  "cacheControl": "public, max-age=3600",
+  "contentEncoding": "identity",
+  "downloadTokens": [
+    "c3c71086-95a8-445d-96e7-f625972de4b0"
+  ],
+  "etag": "PQJQBXRweACX9yRsBEInQjOJ/0s",
+  "timeCreated": "2022-03-24T01:22:20.926Z",
+  "updated": "2022-03-24T01:22:20.926Z",
+  "size": 0,
+  "md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==",
+  "crc32c": "0"
+}

scripts/storage-emulator-integration/import/nested-emulator-data/firebase-export-metadata.json

@@ -0,0 +1,7 @@
+{
+  "version": "10.4.2",
+  "storage": {
+    "version": "10.4.2",
+    "path": "storage_export"
+  }
+}

scripts/storage-emulator-integration/import/nested-emulator-data/storage_export/buckets.json

@@ -0,0 +1,7 @@
+{
+  "buckets": [
+    {
+      "id": "fake-project-id.appspot.com"
+    }
+  ]
+}

scripts/storage-emulator-integration/import/nested-emulator-data/storage_export/metadata/fake-project-id.appspot.com/test_upload.jpg.json

@@ -0,0 +1,20 @@
+{
+  "name": "test_upload.jpg",
+  "bucket": "fake-project-id.appspot.com",
+  "contentType": "application/octet-stream",
+  "metageneration": 1,
+  "generation": 1648084940926,
+  "storageClass": "STANDARD",
+  "contentDisposition": "inline",
+  "cacheControl": "public, max-age=3600",
+  "contentEncoding": "identity",
+  "downloadTokens": [
+    "c3c71086-95a8-445d-96e7-f625972de4b0"
+  ],
+  "etag": "PQJQBXRweACX9yRsBEInQjOJ/0s",
+  "timeCreated": "2022-03-24T01:22:20.926Z",
+  "updated": "2022-03-24T01:22:20.926Z",
+  "size": 0,
+  "md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==",
+  "crc32c": "0"
+}

scripts/storage-emulator-integration/import/tests.ts

@@ -0,0 +1,111 @@
+import * as path from "path";
+import * as fs from "fs-extra";
+import { expect } from "chai";
+import supertest = require("supertest");
+
+import { createTmpDir } from "../../../src/test/emulators/fixtures";
+import { Emulators } from "../../../src/emulator/types";
+import { TriggerEndToEndTest } from "../../integration-helpers/framework";
+import {
+  EMULATORS_SHUTDOWN_DELAY_MS,
+  FIREBASE_EMULATOR_CONFIG,
+  getStorageEmulatorHost,
+  readEmulatorConfig,
+  TEST_SETUP_TIMEOUT,
+} from "../utils";
+
+describe("Import Emulator Data", () => {
+  const FIREBASE_PROJECT = "fake-project-id";
+  const BUCKET = `${FIREBASE_PROJECT}.appspot.com`;
+  const EMULATOR_CONFIG = readEmulatorConfig(FIREBASE_EMULATOR_CONFIG);
+  const STORAGE_EMULATOR_HOST = getStorageEmulatorHost(EMULATOR_CONFIG);
+  const test = new TriggerEndToEndTest(FIREBASE_PROJECT, __dirname, EMULATOR_CONFIG);
+
+  it("retrieves file from imported flattened emulator data", async function (this) {
+    this.timeout(TEST_SETUP_TIMEOUT);
+    await test.startEmulators([
+      "--only",
+      Emulators.STORAGE,
+      "--import",
+      path.join(__dirname, "flattened-emulator-data"),
+    ]);
+
+    await supertest(STORAGE_EMULATOR_HOST)
+      .get(`/v0/b/${BUCKET}/o/test_upload.jpg`)
+      .set({ Authorization: "Bearer owner" })
+      .expect(200);
+  });
+
+  it("stores only the files as blobs when importing emulator data", async function (this) {
+    this.timeout(TEST_SETUP_TIMEOUT);
+    const exportedData = createTmpDir("exported-emulator-data");
+
+    // Import data and export it again on exit
+    await test.startEmulators([
+      "--only",
+      Emulators.STORAGE,
+      "--import",
+      path.join(__dirname, "nested-emulator-data"),
+      "--export-on-exit",
+      exportedData,
+    ]);
+    await test.stopEmulators();
+
+    expect(fs.readdirSync(path.join(exportedData, "storage_export", "blobs")).length).to.equal(1);
+    expect(fs.readdirSync(path.join(exportedData, "storage_export", "blobs"))[0]).to.equal(
+      encodeURIComponent(`${BUCKET}/test_upload.jpg`)
+    );
+  });
+
+  it("retrieves file from imported nested emulator data", async function (this) {
+    this.timeout(TEST_SETUP_TIMEOUT);
+    await test.startEmulators([
+      "--only",
+      Emulators.STORAGE,
+      "--import",
+      path.join(__dirname, "nested-emulator-data"),
+    ]);
+
+    await supertest(STORAGE_EMULATOR_HOST)
+      .get(`/v0/b/${BUCKET}/o/test_upload.jpg`)
+      .set({ Authorization: "Bearer owner" })
+      .expect(200);
+  });
+
+  it("retrieves file from importing previously exported emulator data", async function (this) {
+    this.timeout(TEST_SETUP_TIMEOUT);
+    const exportedData = createTmpDir("exported-emulator-data");
+
+    // Upload file to Storage and export emulator data to tmp directory
+    await test.startEmulators(["--only", Emulators.STORAGE, "--export-on-exit", exportedData]);
+    const uploadURL = await supertest(STORAGE_EMULATOR_HOST)
+      .post(`/v0/b/${BUCKET}/o/test_upload.jpg?uploadType=resumable&name=test_upload.jpg`)
+      .set({
+        Authorization: "Bearer owner",
+        "X-Goog-Upload-Protocol": "resumable",
+        "X-Goog-Upload-Command": "start",
+      })
+      .then((res) => new URL(res.header["x-goog-upload-url"]));
+
+    await supertest(STORAGE_EMULATOR_HOST)
+      .put(uploadURL.pathname + uploadURL.search)
+      .set({
+        "X-Goog-Upload-Protocol": "resumable",
+        "X-Goog-Upload-Command": "upload, finalize",
+      });
+
+    await test.stopEmulators();
+
+    // Import previously exported emulator data and retrieve file from Storage
+    await test.startEmulators(["--only", Emulators.STORAGE, "--import", exportedData]);
+    await supertest(STORAGE_EMULATOR_HOST)
+      .get(`/v0/b/${BUCKET}/o/test_upload.jpg`)
+      .set({ Authorization: "Bearer owner" })
+      .expect(200);
+  });
+
+  afterEach(async function (this) {
+    this.timeout(EMULATORS_SHUTDOWN_DELAY_MS);
+    await test.stopEmulators();
+  });
+});

scripts/storage-emulator-integration/rules/manager.test.ts

@@ -15,23 +15,18 @@ import { RulesetOperationMethod, SourceFile } from "../../../src/emulator/storag
 import { isPermitted } from "../../../src/emulator/storage/rules/utils";
 import { readFile } from "../../../src/fsutils";
 
+const EMULATOR_LOAD_RULESET_DELAY_MS = 2000;
+
 describe("Storage Rules Manager", function () {
   const rulesRuntime = new StorageRulesRuntime();
-  const rules = [
-    { resource: "bucket_0", rules: StorageRulesFiles.readWriteIfTrue },
-    { resource: "bucket_1", rules: StorageRulesFiles.readWriteIfAuth },
-  ];
-  const opts = { method: RulesetOperationMethod.GET, file: {}, path: "/b/bucket_2/o/" };
+  const opts = { method: RulesetOperationMethod.GET, file: {}, path: "/b/bucket_0/o/" };
   let rulesManager: StorageRulesManager;
 
   // eslint-disable-next-line @typescript-eslint/no-invalid-this
   this.timeout(TIMEOUT_LONG);
 
   beforeEach(async () => {
     await rulesRuntime.start();
-
-    rulesManager = createStorageRulesManager(rules, rulesRuntime);
-    await rulesManager.start();
   });
 
   afterEach(async () => {
@@ -40,47 +35,28 @@ describe("Storage Rules Manager", function () {
   });
 
   it("should load multiple rulesets on start", async () => {
+    const rules = [
+      { resource: "bucket_0", rules: StorageRulesFiles.readWriteIfTrue },
+      { resource: "bucket_1", rules: StorageRulesFiles.readWriteIfAuth },
+    ];
+    rulesManager = createStorageRulesManager(rules, rulesRuntime);
+    await rulesManager.start();
+
     const bucket0Ruleset = rulesManager.getRuleset("bucket_0");
-    expect(await isPermitted({ ...opts, ruleset: bucket0Ruleset! })).to.be.true;
+    expect(await isPermitted({ ...opts, path: "/b/bucket_0/o/", ruleset: bucket0Ruleset! })).to.be
+      .true;
 
     const bucket1Ruleset = rulesManager.getRuleset("bucket_1");
-    expect(await isPermitted({ ...opts, ruleset: bucket1Ruleset! })).to.be.false;
+    expect(await isPermitted({ ...opts, path: "/b/bucket_1/o/", ruleset: bucket1Ruleset! })).to.be
+      .false;
   });
 
   it("should load single ruleset on start", async () => {
-    const otherRulesManager = createStorageRulesManager(
-      StorageRulesFiles.readWriteIfTrue,
-      rulesRuntime
-    );
-    await otherRulesManager.start();
+    rulesManager = createStorageRulesManager(StorageRulesFiles.readWriteIfTrue, rulesRuntime);
+    await rulesManager.start();
 
-    const ruleset = otherRulesManager.getRuleset("default");
+    const ruleset = rulesManager.getRuleset("bucket");
     expect(await isPermitted({ ...opts, ruleset: ruleset! })).to.be.true;
-
-    await otherRulesManager.stop();
-  });
-
-  it("should load ruleset on update with SourceFile object", async () => {
-    expect(rulesManager.getRuleset("bucket_2")).to.be.undefined;
-    await rulesManager.updateSourceFile(StorageRulesFiles.readWriteIfTrue, "bucket_2");
-    expect(rulesManager.getRuleset("bucket_2")).not.to.be.undefined;
-  });
-
-  it("should set source file", async () => {
-    await rulesManager.updateSourceFile(StorageRulesFiles.readWriteIfTrue, "bucket_2");
-
-    expect(await isPermitted({ ...opts, ruleset: rulesManager.getRuleset("bucket_2")! })).to.be
-      .true;
-
-    const issues = await rulesManager.updateSourceFile(
-      StorageRulesFiles.readWriteIfAuth,
-      "bucket_2"
-    );
-
-    expect(issues.errors.length).to.equal(0);
-    expect(issues.warnings.length).to.equal(0);
-    expect(await isPermitted({ ...opts, ruleset: rulesManager.getRuleset("bucket_2")! })).to.be
-      .false;
   });
 
   it("should reload ruleset on changes to source file", async () => {
@@ -91,15 +67,17 @@ describe("Storage Rules Manager", function () {
     persistence.appendBytes(fileName, Buffer.from(StorageRulesFiles.readWriteIfTrue.content));
 
     const sourceFile = getSourceFile(testDir, fileName);
-    await rulesManager.updateSourceFile(sourceFile, "bucket_2");
-    expect(await isPermitted({ ...opts, ruleset: rulesManager.getRuleset("bucket_2")! })).to.be
-      .true;
+    rulesManager = createStorageRulesManager(sourceFile, rulesRuntime);
+    await rulesManager.start();
+
+    expect(await isPermitted({ ...opts, ruleset: rulesManager.getRuleset("bucket")! })).to.be.true;
 
     // Write new rules to file
     persistence.deleteFile(fileName);
     persistence.appendBytes(fileName, Buffer.from(StorageRulesFiles.readWriteIfAuth.content));
 
-    expect(await isPermitted(opts)).to.be.false;
+    await new Promise((resolve) => setTimeout(resolve, EMULATOR_LOAD_RULESET_DELAY_MS));
+    expect(await isPermitted({ ...opts, ruleset: rulesManager.getRuleset("bucket")! })).to.be.false;
   });
 });