Skip to content

Commit

Permalink
Corrected list objects endpoint when using prefix and delimiter to re…
Browse files Browse the repository at this point in the history 
…turn correct items and include prefixes
  • Loading branch information
@rhodgkins
rhodgkins committed Aug 10, 2021
1 parent d44df25 commit 035ff8d
Show file tree
Hide file tree
Showing 3 changed files with 215 additions and 36 deletions.
184 changes: 178 additions & 6 deletions scripts/storage-emulator-integration/tests.ts
View file
Expand Up @@ -259,16 +259,188 @@ describe("Storage emulator", () => {
});

describe("#getFiles()", () => {
it("should list files", async () => {
await testBucket.upload(smallFilePath, {
destination: "testing/shoveler.svg",
const TESTING_FILE = "testing/shoveler.svg";
const PREFIX_FILE = "prefix";
const PREFIX_1_FILE = PREFIX_FILE + "/1.txt";
const PREFIX_2_FILE = PREFIX_FILE + "/2.txt";
const PREFIX_SUB_DIRECTORY_FILE = PREFIX_FILE + "/dir/file.txt";

beforeEach(async () => {
await Promise.all(
[
TESTING_FILE,
PREFIX_FILE,
PREFIX_1_FILE,
PREFIX_2_FILE,
PREFIX_SUB_DIRECTORY_FILE,
].map(async (f) => {
await testBucket.upload(smallFilePath, {
destination: f,
});
})
);
});

it("should list all files in bucket", async () => {
// This is only test that uses autoPagination as the other tests look at the prefixes response
const [files] = await testBucket.getFiles();

expect(files.map((file) => file.name)).to.deep.equal([
PREFIX_FILE,
PREFIX_1_FILE,
PREFIX_2_FILE,
PREFIX_SUB_DIRECTORY_FILE,
TESTING_FILE,
]);
});

it("should list files with prefix", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
prefix: "prefix",
});
const [files, prefixes] = await testBucket.getFiles({
directory: "testing",

expect(prefixes).to.be.undefined;
expect(files.map((file) => file.name)).to.deep.equal([
PREFIX_FILE,
PREFIX_1_FILE,
PREFIX_2_FILE,
PREFIX_SUB_DIRECTORY_FILE,
]);
});

it("should list files using common delimiter", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
delimiter: "/",
});

expect(prefixes).to.be.deep.equal(["prefix/", "testing/"]);
expect(files.map((file) => file.name)).to.deep.equal([PREFIX_FILE]);
});

it("should list files using other delimiter", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
delimiter: "dir",
});

expect(prefixes).to.be.deep.equal(["prefix/dir"]);
expect(files.map((file) => file.name)).to.deep.equal([
PREFIX_FILE,
PREFIX_1_FILE,
PREFIX_2_FILE,
TESTING_FILE,
]);
});

it("should list files using same prefix and delimiter of p", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
prefix: "p",
delimiter: "p",
});

expect(prefixes).to.be.undefined;
expect(files.map((file) => file.name)).to.deep.equal([
PREFIX_FILE,
PREFIX_1_FILE,
PREFIX_2_FILE,
PREFIX_SUB_DIRECTORY_FILE,
]);
});

it("should list files using same prefix and delimiter of t", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
prefix: "t",
delimiter: "t",
});

expect(prefixes).to.be.deep.equal(["test"]);
expect(files.map((file) => file.name)).to.be.empty;
});

it("should list files using prefix=p and delimiter=t", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
prefix: "p",
delimiter: "t",
});

expect(prefixes).to.be.deep.equal(["prefix/1.t", "prefix/2.t", "prefix/dir/file.t"]);
expect(files.map((file) => file.name)).to.deep.equal([PREFIX_FILE]);
});

it("should list files in sub-directory (using prefix and delimiter)", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
prefix: "prefix/",
delimiter: "/",
});

expect(prefixes).to.be.deep.equal(["prefix/dir/"]);
expect(files.map((file) => file.name)).to.deep.equal([PREFIX_1_FILE, PREFIX_2_FILE]);
});

it("should list files in sub-directory (using prefix)", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
prefix: "prefix/",
});

expect(prefixes).to.be.undefined;
expect(files.map((file) => file.name)).to.deep.equal([
PREFIX_1_FILE,
PREFIX_2_FILE,
PREFIX_SUB_DIRECTORY_FILE,
]);
});

it("should list files in sub-directory (using directory)", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
directory: "testing/",
});

expect(prefixes).to.be.undefined;
expect(files.map((file) => file.name)).to.deep.equal(["testing/shoveler.svg"]);
expect(files.map((file) => file.name)).to.deep.equal([TESTING_FILE]);
});

it("should list no files for unused prefix", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
prefix: "blah/",
});

expect(prefixes).to.be.undefined;
expect(files).to.be.empty;
});

it("should list files using prefix=pref and delimiter=i", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
prefix: "pref",
delimiter: "i",
});

expect(prefixes).to.be.deep.equal(["prefi"]);
expect(files).to.be.empty;
});

it("should list files using prefix=prefi and delimiter=i", async () => {
const [files, , { prefixes }] = await testBucket.getFiles({
autoPaginate: false,
prefix: "prefi",
delimiter: "i",
});

expect(prefixes).to.be.deep.equal(["prefix/di"]);
expect(files.map((file) => file.name)).to.deep.equal([
PREFIX_FILE,
PREFIX_1_FILE,
PREFIX_2_FILE,
]);
});
});
});
Expand Down
2 changes: 1 addition & 1 deletion src/emulator/storage/apis/gcloud.ts
View file
Expand Up @@ -76,7 +76,7 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
if (req.query.maxResults) {
maxRes = +req.query.maxResults.toString();
}
const delimiter = req.query.delimiter ? req.query.delimiter.toString() : "/";
const delimiter = req.query.delimiter ? req.query.delimiter.toString() : "";
const pageToken = req.query.pageToken ? req.query.pageToken.toString() : undefined;
const prefix = req.query.prefix ? req.query.prefix.toString() : "";

Expand Down
65 changes: 36 additions & 29 deletions src/emulator/storage/files.ts
View file
Expand Up @@ -401,41 +401,53 @@ export class StorageLayer {
delimiter: string,
pageToken: string | undefined,
maxResults: number | undefined
) {
if (!delimiter) {
delimiter = "/";
}

if (!prefix) {
prefix = "";
}

if (!prefix.endsWith(delimiter)) {
prefix += delimiter;
}

let items = [];
): {
kind: string;
prefixes?: string[];
items?: CloudStorageObjectMetadata[];
nextPageToken?: string;
} {
let items: Array<StoredFileMetadata> = [];
const prefixes = new Set<string>();
for (const [, file] of this._files) {
if (file.metadata.bucket != bucket) {
continue;
}

let name = file.metadata.name;
const name = file.metadata.name;
if (!name.startsWith(prefix)) {
continue;
}

name = name.substring(prefix.length);
if (name.startsWith(delimiter)) {
name = name.substring(prefix.length);
let includeMetadata = true;
if (delimiter) {
const delimiterIdx = name.indexOf(delimiter);
const delimiterAfterPrefixIdx = name.indexOf(delimiter, prefix.length);
// items[] contains object metadata for objects whose names do not contain delimiter, or whose names only have instances of delimiter in their prefix.
includeMetadata = delimiterIdx === -1 || delimiterAfterPrefixIdx === -1;
if (delimiterAfterPrefixIdx !== -1) {
// prefixes[] contains truncated object names for objects whose names contain delimiter after any prefix. Object names are truncated beyond the first applicable instance of the delimiter.
prefixes.add(name.slice(0, delimiterAfterPrefixIdx + delimiter.length));
}
}

items.push(this.path(file.metadata.bucket, file.metadata.name));
if (includeMetadata) {
items.push(file.metadata);
}
}

items.sort();
// Order items by name
items.sort((a, b) => {
if (a.name === b.name) {
return 0;
} else if (a.name < b.name) {
return -1;
} else {
return 1;
}
});
if (pageToken) {
const idx = items.findIndex((v) => v == pageToken);
const idx = items.findIndex((v) => v.name == pageToken);
if (idx != -1) {
items = items.slice(idx);
}
Expand All @@ -447,14 +459,9 @@ export class StorageLayer {

return {
kind: "#storage/objects",
items: items.map((item) => {
const storedFile = this._files.get(item);
if (!storedFile) {
return console.warn(`No file ${item}`);
}

return new CloudStorageObjectMetadata(storedFile.metadata);
}),
prefixes: prefixes.size > 0 ? [...prefixes].sort() : undefined,
items:
items.length > 0 ? items.map((item) => new CloudStorageObjectMetadata(item)) : undefined,
};
}

Expand Down

0 comments on commit 035ff8d

Please sign in to comment.