-
Notifications
You must be signed in to change notification settings - Fork 903
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Delete Container Registry images left after Functions deployment (#3439)
* Delete Container Registry images left after Functions deployment * Simplify caching * Improve error handling and report next steps to users * lint fixes * Fix typo
- Loading branch information
Showing
5 changed files
with
515 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,209 @@ | ||
// This code is very aggressive about running requests in parallel and does not use | ||
// a task queue, because the quota limits for GCR.io are absurdly high. At the time | ||
// of writing, we can make 50K requests per 10m. | ||
// https://cloud.google.com/container-registry/quotas | ||
|
||
import * as clc from "cli-color"; | ||
|
||
import { containerRegistryDomain } from "../../api"; | ||
import { logger } from "../../logger"; | ||
import * as docker from "../../gcp/docker"; | ||
import * as backend from "./backend"; | ||
import * as utils from "../../utils"; | ||
|
||
// A flattening of container_registry_hosts and | ||
// region_multiregion_map from regionconfig.borg | ||
const SUBDOMAIN_MAPPING: Record<string, string> = { | ||
"us-west2": "us", | ||
"us-west3": "us", | ||
"us-west4": "us", | ||
"us-central1": "us", | ||
"us-central2": "us", | ||
"us-east1": "us", | ||
"us-east4": "us", | ||
"northamerica-northeast1": "us", | ||
"southamerica-east1": "us", | ||
"europe-west1": "eu", | ||
"europe-west2": "eu", | ||
"europe-west3": "eu", | ||
"europe-west5": "eu", | ||
"europe-west6": "eu", | ||
"europe-central2": "eu", | ||
"asia-east1": "asia", | ||
"asia-east2": "asia", | ||
"asia-northeast1": "asia", | ||
"asia-northeast2": "asia", | ||
"asia-northeast3": "asia", | ||
"asia-south1": "asia", | ||
"asia-southeast2": "asia", | ||
"australia-southeast1": "asia", | ||
}; | ||
|
||
export async function cleanupBuildImages(functions: backend.FunctionSpec[]): Promise<void> { | ||
utils.logBullet(clc.bold.cyan("functions: ") + "cleaning up build files..."); | ||
const gcrCleaner = new ContainerRegistryCleaner(); | ||
const failedDomains: Set<string> = new Set(); | ||
await Promise.all( | ||
functions.map((func) => | ||
(async () => { | ||
try { | ||
await gcrCleaner.cleanupFunction(func); | ||
} catch (err) { | ||
const path = `${func.project}/${SUBDOMAIN_MAPPING[func.region]}/gcf`; | ||
failedDomains.add(`https://console.cloud.google.com/gcr/images/${path}`); | ||
} | ||
})() | ||
) | ||
); | ||
if (failedDomains.size) { | ||
let message = | ||
"Unhandled error cleaning up build images. This could result in a small monthly bill if not corrected. "; | ||
message += | ||
"You can attempt to delete these images by redeploying or you can delete them manually at"; | ||
if (failedDomains.size == 1) { | ||
message += " " + failedDomains.values().next().value; | ||
} else { | ||
message += [...failedDomains].map((domain) => "\n\t" + domain).join(""); | ||
} | ||
utils.logLabeledWarning("functions", message); | ||
} | ||
|
||
// TODO: clean up Artifact Registry images as well. | ||
} | ||
|
||
export class ContainerRegistryCleaner { | ||
readonly helpers: Record<string, DockerHelper> = {}; | ||
|
||
private helper(location: string): DockerHelper { | ||
const subdomain = SUBDOMAIN_MAPPING[location] || "us"; | ||
if (!this.helpers[subdomain]) { | ||
const origin = `https://${subdomain}.${containerRegistryDomain}`; | ||
this.helpers[subdomain] = new DockerHelper(origin); | ||
} | ||
return this.helpers[subdomain]; | ||
} | ||
|
||
// GCFv1 has the directory structure: | ||
// gcf/ | ||
// +- <region>/ | ||
// +- <uuid> | ||
// +- <hash> (tags: <FuncName>_version-<#>) | ||
// +- cache/ (Only present in first deploy of region) | ||
// | +- <hash> (tags: latest) | ||
// +- worker/ (Only present in first deploy of region) | ||
// +- <hash> (tags: latest) | ||
// | ||
// We'll parallel search for the valid <uuid> and their children | ||
// until we find one with the right tag for the function name. | ||
// The underlying Helper's caching should make this expensive for | ||
// the first function and free for the next functions in the same | ||
// region. | ||
async cleanupFunction(func: backend.FunctionSpec): Promise<void> { | ||
const helper = this.helper(func.region); | ||
const uuids = (await helper.ls(`${func.project}/gcf/${func.region}`)).children; | ||
|
||
const uuidTags: Record<string, string[]> = {}; | ||
const loadUuidTags: Promise<void>[] = []; | ||
for (const uuid of uuids) { | ||
loadUuidTags.push( | ||
(async () => { | ||
const path = `${func.project}/gcf/${func.region}/${uuid}`; | ||
const tags = (await helper.ls(path)).tags; | ||
uuidTags[path] = tags; | ||
})() | ||
); | ||
} | ||
await Promise.all(loadUuidTags); | ||
|
||
const extractFunction = /^(.*)_version-\d+$/; | ||
const entry = Object.entries(uuidTags).find(([, tags]) => { | ||
return tags.find((tag) => extractFunction.exec(tag)?.[1] === func.id); | ||
}); | ||
|
||
if (!entry) { | ||
logger.debug("Could not find image for function", backend.functionName(func)); | ||
return; | ||
} | ||
await helper.rm(entry[0]); | ||
} | ||
} | ||
|
||
export interface Stat { | ||
children: string[]; | ||
digests: docker.Digest[]; | ||
tags: docker.Tag[]; | ||
} | ||
|
||
export class DockerHelper { | ||
readonly client: docker.Client; | ||
readonly cache: Record<string, Stat> = {}; | ||
|
||
constructor(origin: string) { | ||
this.client = new docker.Client(origin); | ||
} | ||
|
||
async ls(path: string): Promise<Stat> { | ||
if (!this.cache[path]) { | ||
const raw = await this.client.listTags(path); | ||
this.cache[path] = { | ||
tags: raw.tags, | ||
digests: Object.keys(raw.manifest), | ||
children: raw.child, | ||
}; | ||
} | ||
return this.cache[path]; | ||
} | ||
|
||
// While we can't guarantee all promises will succeed, we can do our darndest | ||
// to expunge as much as possible before throwing. | ||
async rm(path: string): Promise<void> { | ||
let toThrowLater: any = undefined; | ||
const stat = await this.ls(path); | ||
const recursive = stat.children.map((child) => | ||
(async () => { | ||
try { | ||
await this.rm(`${path}/${child}`); | ||
stat.children.splice(stat.children.indexOf(child), 1); | ||
} catch (err) { | ||
toThrowLater = err; | ||
} | ||
})() | ||
); | ||
// Unlike a filesystem, we can delete a "directory" while its children are still being | ||
// deleted. Run these in parallel to improve performance and just wait for the result | ||
// before the function's end. | ||
|
||
// An image cannot be deleted until its tags have been removed. Do this in two phases. | ||
const deleteTags = stat.tags.map((tag) => | ||
(async () => { | ||
try { | ||
await this.client.deleteTag(path, tag); | ||
stat.tags.splice(stat.tags.indexOf(tag), 1); | ||
} catch (err) { | ||
logger.debug("Got error trying to remove docker tag:", err); | ||
toThrowLater = err; | ||
} | ||
})() | ||
); | ||
await Promise.all(deleteTags); | ||
|
||
const deleteImages = stat.digests.map((digest) => | ||
(async () => { | ||
try { | ||
await this.client.deleteImage(path, digest); | ||
stat.digests.splice(stat.digests.indexOf(digest), 1); | ||
} catch (err) { | ||
logger.debug("Got error trying to remove docker image:", err); | ||
toThrowLater = err; | ||
} | ||
})() | ||
); | ||
await Promise.all(deleteImages); | ||
|
||
await Promise.all(recursive); | ||
|
||
if (toThrowLater) { | ||
throw toThrowLater; | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
// Note: unlike Google APIs, the documentation for the GCR API is | ||
// actually the Docker REST API. This can be found at | ||
// https://docs.docker.com/registry/spec/api/ | ||
// This API is _very_ complex in its entirety and is very subtle (e.g. tags and digests | ||
// are both strings and can both be put in the same route to get completely different | ||
// response document types). | ||
// This file will only implement a minimal subset as needed. | ||
import { FirebaseError } from "../error"; | ||
import * as api from "../apiv2"; | ||
|
||
// A Digest is a string in the format <algorithm>:<hex>. For example: | ||
// sha256:146d8c9dff0344fb01417ef28673ed196e38215f3c94837ae733d3b064ba439e | ||
export type Digest = string; | ||
export type Tag = string; | ||
|
||
export interface Tags { | ||
name: string; | ||
tags: string[]; | ||
|
||
// These fields are not documented in the Docker API but are | ||
// present in the GCR API. | ||
manifest: Record<Digest, ImageInfo>; | ||
child: string[]; | ||
} | ||
|
||
export interface ImageInfo { | ||
// times are string milliseconds | ||
timeCreatedMs: string; | ||
timeUploadedMs: string; | ||
tag: string[]; | ||
mediaType: string; | ||
imageSizeBytes: string; | ||
layerId: string; | ||
} | ||
|
||
interface ErrorsResponse { | ||
errors?: { | ||
code: string; | ||
message: string; | ||
details: unknown; | ||
}[]; | ||
} | ||
|
||
function isErrors(response: unknown): response is ErrorsResponse { | ||
return Object.prototype.hasOwnProperty.call(response, "errors"); | ||
} | ||
|
||
const API_VERSION = "v2"; | ||
|
||
export class Client { | ||
readonly client: api.Client; | ||
|
||
constructor(origin: string) { | ||
this.client = new api.Client({ | ||
apiVersion: API_VERSION, | ||
auth: true, | ||
urlPrefix: origin, | ||
}); | ||
} | ||
|
||
async listTags(path: string): Promise<Tags> { | ||
const response = await this.client.get<Tags | ErrorsResponse>(`${path}/tags/list`); | ||
if (isErrors(response.body)) { | ||
throw new FirebaseError(`Failed to list GCR tags at ${path}`, { | ||
children: response.body.errors, | ||
}); | ||
} | ||
return response.body; | ||
} | ||
|
||
async deleteTag(path: string, tag: Tag): Promise<void> { | ||
const response = await this.client.delete<ErrorsResponse>(`${path}/manifests/${tag}`); | ||
if (response.body.errors?.length != 0) { | ||
throw new FirebaseError(`Failed to delete tag ${tag} at path ${path}`, { | ||
children: response.body.errors, | ||
}); | ||
} | ||
} | ||
|
||
async deleteImage(path: string, digest: Digest): Promise<void> { | ||
const response = await this.client.delete<ErrorsResponse>(`${path}/manifests/${digest}`); | ||
if (response.body.errors?.length != 0) { | ||
throw new FirebaseError(`Failed to delete image ${digest} at path ${path}`, { | ||
children: response.body.errors, | ||
}); | ||
} | ||
} | ||
} |
Oops, something went wrong.