Skip to content

Commit

Permalink
Add StorageRulesManagerRegistry to handle rules files for multiple ta…
Browse files Browse the repository at this point in the history
…rgets (#4281)

* Address PR feedback; add JSDocs for new method

* Make StorageRulesManager an interface; add StorageRulesManagerRegistry

* Modify RulesetProvider to take resource parameter

* Rebase; fix lint error

* Change emulator arg to take SourceFile only

* PR feedback, mostly re: API usage

* Don't delete source file or ruleset on stop(); add CHANGELOG

* Address PR feedback

* Moved Storage rules integration tests under scripts/ (#4309)

* Make StorageRulesManager an interface; add StorageRulesManagerRegistry

* Modify RulesetProvider to take resource parameter

* Change emulator arg to take SourceFile only

* PR feedback, mostly re: API usage

* Don't delete source file or ruleset on stop(); add CHANGELOG

* Address PR feedback

* Fix lint

* Add test for failing to find rules for given resource
  • Loading branch information
tohhsinpei committed Mar 16, 2022
1 parent 1322164 commit c69b437
Show file tree
Hide file tree
Showing 13 changed files with 286 additions and 144 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Adds support for configuration with multiple storage targets (#4281).
117 changes: 67 additions & 50 deletions scripts/storage-emulator-integration/rules/manager.test.ts
Original file line number Diff line number Diff line change
@@ -1,92 +1,109 @@
import { expect } from "chai";
import { tmpdir } from "os";
import { v4 as uuidv4 } from "uuid";

import { FirebaseError } from "../../../src/error";
import { StorageRulesFiles, TIMEOUT_MED } from "../../../src/test/emulators/fixtures";
import { StorageRulesManager } from "../../../src/emulator/storage/rules/manager";
import {
createTmpDir,
StorageRulesFiles,
TIMEOUT_LONG,
} from "../../../src/test/emulators/fixtures";
import {
createStorageRulesManager,
StorageRulesManager,
} from "../../../src/emulator/storage/rules/manager";
import { StorageRulesRuntime } from "../../../src/emulator/storage/rules/runtime";
import { Persistence } from "../../../src/emulator/storage/persistence";
import { RulesetOperationMethod } from "../../../src/emulator/storage/rules/types";
import { RulesetOperationMethod, SourceFile } from "../../../src/emulator/storage/rules/types";
import { isPermitted } from "../../../src/emulator/storage/rules/utils";
import { readFile } from "../../../src/fsutils";

describe("Storage Rules Manager", function () {
const rulesRuntime = new StorageRulesRuntime();
const rulesManager = new StorageRulesManager(rulesRuntime);
const rules = [
{ resource: "bucket_0", rules: StorageRulesFiles.readWriteIfTrue },
{ resource: "bucket_1", rules: StorageRulesFiles.readWriteIfAuth },
];
const opts = { method: RulesetOperationMethod.GET, file: {}, path: "/b/bucket_2/o/" };
let rulesManager: StorageRulesManager;

// eslint-disable-next-line @typescript-eslint/no-invalid-this
this.timeout(TIMEOUT_MED);
this.timeout(TIMEOUT_LONG);

before(async () => {
beforeEach(async () => {
await rulesRuntime.start();

rulesManager = createStorageRulesManager(rules, rulesRuntime);
await rulesManager.start();
});

after(async () => {
afterEach(async () => {
rulesRuntime.stop();
await rulesManager.close();
await rulesManager.stop();
});

it("should load ruleset from SourceFile object", async () => {
await rulesManager.setSourceFile(StorageRulesFiles.readWriteIfTrue);
expect(rulesManager.ruleset).not.to.be.undefined;
it("should load multiple rulesets on start", async () => {
const bucket0Ruleset = rulesManager.getRuleset("bucket_0");
expect(await isPermitted({ ...opts, ruleset: bucket0Ruleset! })).to.be.true;

const bucket1Ruleset = rulesManager.getRuleset("bucket_1");
expect(await isPermitted({ ...opts, ruleset: bucket1Ruleset! })).to.be.false;
});

it("should load ruleset from file path", async () => {
// Write rules to file
const fileName = "storage.rules";
const testDir = `${tmpdir()}/${uuidv4()}`;
const persistence = new Persistence(testDir);
persistence.appendBytes(fileName, Buffer.from(StorageRulesFiles.readWriteIfTrue.content));
it("should load single ruleset on start", async () => {
const otherRulesManager = createStorageRulesManager(
StorageRulesFiles.readWriteIfTrue,
rulesRuntime
);
await otherRulesManager.start();

await rulesManager.setSourceFile(`${testDir}/${fileName}`);
const ruleset = otherRulesManager.getRuleset("default");
expect(await isPermitted({ ...opts, ruleset: ruleset! })).to.be.true;

expect(rulesManager.ruleset).not.to.be.undefined;
await otherRulesManager.stop();
});

it("should load ruleset on update with SourceFile object", async () => {
expect(rulesManager.getRuleset("bucket_2")).to.be.undefined;
await rulesManager.updateSourceFile(StorageRulesFiles.readWriteIfTrue, "bucket_2");
expect(rulesManager.getRuleset("bucket_2")).not.to.be.undefined;
});

it("should set source file", async () => {
await rulesManager.setSourceFile(StorageRulesFiles.readWriteIfTrue);
const opts = { method: RulesetOperationMethod.GET, file: {}, path: "/b/bucket/o/" };
expect((await rulesManager.ruleset!.verify(opts)).permitted).to.be.true;
await rulesManager.updateSourceFile(StorageRulesFiles.readWriteIfTrue, "bucket_2");

const issues = await rulesManager.setSourceFile(StorageRulesFiles.readWriteIfAuth);
expect(await isPermitted({ ...opts, ruleset: rulesManager.getRuleset("bucket_2")! })).to.be
.true;

const issues = await rulesManager.updateSourceFile(
StorageRulesFiles.readWriteIfAuth,
"bucket_2"
);

expect(issues.errors.length).to.equal(0);
expect(issues.warnings.length).to.equal(0);
expect((await rulesManager.ruleset!.verify(opts)).permitted).to.be.false;
expect(await isPermitted({ ...opts, ruleset: rulesManager.getRuleset("bucket_2")! })).to.be
.false;
});

it("should reload ruleset on changes to source file", async () => {
const opts = { method: RulesetOperationMethod.GET, file: {}, path: "/b/bucket/o/" };

// Write rules to file
const fileName = "storage.rules";
const testDir = `${tmpdir()}/${uuidv4()}`;
const testDir = createTmpDir("storage-files");
const persistence = new Persistence(testDir);
persistence.appendBytes(fileName, Buffer.from(StorageRulesFiles.readWriteIfTrue.content));

await rulesManager.setSourceFile(`${testDir}/${fileName}`);
expect((await rulesManager.ruleset!.verify(opts)).permitted).to.be.true;
const sourceFile = getSourceFile(testDir, fileName);
await rulesManager.updateSourceFile(sourceFile, "bucket_2");
expect(await isPermitted({ ...opts, ruleset: rulesManager.getRuleset("bucket_2")! })).to.be
.true;

// Write new rules to file
persistence.deleteFile(fileName);
persistence.appendBytes(fileName, Buffer.from(StorageRulesFiles.readWriteIfAuth.content));

await rulesManager.setSourceFile(`${testDir}/${fileName}`);
expect((await rulesManager.ruleset!.verify(opts)).permitted).to.be.false;
});

it("should throw FirebaseError when attempting to set invalid source file", async () => {
const invalidFileName = "foo";
await expect(rulesManager.setSourceFile(invalidFileName)).to.be.rejectedWith(
FirebaseError,
`File not found: ${invalidFileName}`
);
});

it("should delete ruleset when storage manager is closed", async () => {
await rulesManager.setSourceFile(StorageRulesFiles.readWriteIfTrue);
expect(rulesManager.ruleset).not.to.be.undefined;

await rulesManager.close();
expect(rulesManager.ruleset).to.be.undefined;
expect(await isPermitted(opts)).to.be.false;
});
});

function getSourceFile(testDir: string, fileName: string): SourceFile {
const filePath = `${testDir}/${fileName}`;
return { name: filePath, content: readFile(filePath) };
}
14 changes: 5 additions & 9 deletions src/emulator/storage/apis/firebase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,11 @@ export function createFirebaseEndpoints(emulator: StorageEmulator): Router {
});
}

firebaseStorageAPI.use((req, res, next) => {
if (!emulator.rules) {
// Automatically create a bucket for any route which uses a bucket
firebaseStorageAPI.use(/.*\/b\/(.+?)\/.*/, (req, res, next) => {
const bucketId = req.params[0];
storageLayer.createBucket(bucketId);
if (!emulator.rulesManager.getRuleset(bucketId)) {
EmulatorLogger.forEmulator(Emulators.STORAGE).log(
"WARN",
"Permission denied because no Storage ruleset is currently loaded, check your rules for syntax errors."
Expand All @@ -77,13 +80,6 @@ export function createFirebaseEndpoints(emulator: StorageEmulator): Router {
},
});
}

next();
});

// Automatically create a bucket for any route which uses a bucket
firebaseStorageAPI.use(/.*\/b\/(.+?)\/.*/, (req, res, next) => {
storageLayer.createBucket(req.params[0]);
next();
});

Expand Down
5 changes: 5 additions & 0 deletions src/emulator/storage/files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ export class StorageLayer {
if (!authorized) {
authorized = await this._rulesValidator.validate(
["b", request.bucketId, "o", request.decodedObjectId].join("/"),
request.bucketId,
RulesetOperationMethod.GET,
{ before: metadata?.asRulesResource() },
request.authorization
Expand Down Expand Up @@ -213,6 +214,7 @@ export class StorageLayer {
skipAuth ||
(await this._rulesValidator.validate(
["b", request.bucketId, "o", request.decodedObjectId].join("/"),
request.bucketId,
RulesetOperationMethod.DELETE,
{ before: storedMetadata?.asRulesResource() },
request.authorization
Expand Down Expand Up @@ -267,6 +269,7 @@ export class StorageLayer {
skipAuth ||
(await this._rulesValidator.validate(
["b", request.bucketId, "o", request.decodedObjectId].join("/"),
request.bucketId,
RulesetOperationMethod.UPDATE,
{
before: storedMetadata?.asRulesResource(),
Expand Down Expand Up @@ -315,6 +318,7 @@ export class StorageLayer {
skipAuth ||
(await this._rulesValidator.validate(
["b", upload.bucketId, "o", upload.objectId].join("/"),
upload.bucketId,
RulesetOperationMethod.CREATE,
{ after: metadata?.asRulesResource() },
upload.authorization
Expand Down Expand Up @@ -399,6 +403,7 @@ export class StorageLayer {
skipAuth ||
(await this._rulesValidator.validate(
["b", request.bucketId, "o", request.prefix].join("/"),
request.bucketId,
RulesetOperationMethod.LIST,
{},
request.authorization
Expand Down
29 changes: 13 additions & 16 deletions src/emulator/storage/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import { EmulatorInfo, EmulatorInstance, Emulators } from "../types";
import { createApp } from "./server";
import { StorageLayer } from "./files";
import { EmulatorLogger } from "../emulatorLogger";
import { StorageRulesManager } from "./rules/manager";
import { StorageRulesetInstance, StorageRulesRuntime, StorageRulesIssues } from "./rules/runtime";
import { createStorageRulesManager, StorageRulesManager } from "./rules/manager";
import { StorageRulesRuntime } from "./rules/runtime";
import { SourceFile } from "./rules/types";
import express = require("express");
import { getAdminCredentialValidator, getRulesValidator } from "./rules/utils";
Expand All @@ -15,14 +15,17 @@ import { UploadService } from "./upload";

export type RulesConfig = {
resource: string;
rules: string;
rules: SourceFile;
};

export interface StorageEmulatorArgs {
projectId: string;
port?: number;
host?: string;
rules: RulesConfig[];

// Either a single set of rules to be applied to all resources or a mapping of resource to rules
rules: SourceFile | RulesConfig[];

auto_download?: boolean;
}

Expand All @@ -39,11 +42,11 @@ export class StorageEmulator implements EmulatorInstance {

constructor(private args: StorageEmulatorArgs) {
this._rulesRuntime = new StorageRulesRuntime();
this._rulesManager = new StorageRulesManager(this._rulesRuntime);
this._rulesManager = createStorageRulesManager(this.args.rules, this._rulesRuntime);
this._persistence = new Persistence(this.getPersistenceTmpDir());
this._storageLayer = new StorageLayer(
args.projectId,
getRulesValidator(() => this.rules),
getRulesValidator((resource: string) => this._rulesManager.getRuleset(resource)),
getAdminCredentialValidator(),
this._persistence
);
Expand All @@ -58,8 +61,8 @@ export class StorageEmulator implements EmulatorInstance {
return this._uploadService;
}

get rules(): StorageRulesetInstance | undefined {
return this._rulesManager.ruleset;
get rulesManager(): StorageRulesManager {
return this._rulesManager;
}

get logger(): EmulatorLogger {
Expand All @@ -75,9 +78,7 @@ export class StorageEmulator implements EmulatorInstance {
async start(): Promise<void> {
const { host, port } = this.getInfo();
await this._rulesRuntime.start(this.args.auto_download);

// TODO(hsinpei): set source file for multiple resources
await this._rulesManager.setSourceFile(this.args.rules[0].rules);
await this._rulesManager.start();
this._app = await createApp(this.args.projectId, this);
const server = this._app.listen(port, host);
this.destroyServer = utils.createDestroyer(server);
Expand All @@ -87,13 +88,9 @@ export class StorageEmulator implements EmulatorInstance {
// No-op
}

async setRules(rules: SourceFile): Promise<StorageRulesIssues> {
return this._rulesManager.setSourceFile(rules);
}

async stop(): Promise<void> {
await this._persistence.deleteAll();
await this._rulesManager.close();
await this._rulesManager.stop();
return this.destroyServer ? this.destroyServer() : Promise.resolve();
}

Expand Down
28 changes: 19 additions & 9 deletions src/emulator/storage/rules/config.ts
Original file line number Diff line number Diff line change
@@ -1,33 +1,43 @@
import { RulesConfig } from "..";
import { FirebaseError } from "../../../error";
import { readFile } from "../../../fsutils";
import { Options } from "../../../options";
import { SourceFile } from "./types";

function getAbsoluteRulesPath(rules: string, options: Options): string {
return options.config.path(rules);
function getSourceFile(rules: string, options: Options): SourceFile {
const path = options.config.path(rules);
return { name: path, content: readFile(path) };
}

export function getStorageRulesConfig(projectId: string, options: Options): RulesConfig[] {
/**
* Parses rules file for each target specified in the storage config under {@link options}.
* @returns The rules file path if the storage config does not specify a target and an array
* of project resources and their corresponding rules files otherwise.
* @throws {FirebaseError} if storage config is missing or rules file is missing or invalid.
*/
export function getStorageRulesConfig(
projectId: string,
options: Options
): SourceFile | RulesConfig[] {
const storageConfig = options.config.data.storage;
if (!storageConfig) {
throw new FirebaseError(
"Cannot start the Storage emulator without rules file specified in firebase.json: run 'firebase init' and set up your Storage configuration"
);
}

// Single resource
// No target specified
if (!Array.isArray(storageConfig)) {
if (!storageConfig.rules) {
throw new FirebaseError(
"Cannot start the Storage emulator without rules file specified in firebase.json: run 'firebase init' and set up your Storage configuration"
);
}

// TODO(hsinpei): set default resource
const resource = "default";
return [{ resource, rules: getAbsoluteRulesPath(storageConfig.rules, options) }];
return getSourceFile(storageConfig.rules, options);
}

// Multiple resources
// Multiple targets
const results: RulesConfig[] = [];
const { rc } = options;
for (const targetConfig of storageConfig) {
Expand All @@ -36,7 +46,7 @@ export function getStorageRulesConfig(projectId: string, options: Options): Rule
}
rc.requireTarget(projectId, "storage", targetConfig.target);
rc.target(projectId, "storage", targetConfig.target).forEach((resource: string) => {
results.push({ resource, rules: getAbsoluteRulesPath(targetConfig.rules, options) });
results.push({ resource, rules: getSourceFile(targetConfig.rules, options) });
});
}
return results;
Expand Down

0 comments on commit c69b437

Please sign in to comment.