Delete Container Registry images left after Functions deployment (fir…

…ebase#3439)

* Delete Container Registry images left after Functions deployment

* Simplify caching

* Improve error handling and report next steps to users

* lint fixes

* Fix typo

src/api.js

@@ -99,6 +99,7 @@ var api = {
     "FIREBASE_CLOUDLOGGING_URL",
     "https://logging.googleapis.com"
   ),
+  containerRegistryDomain: utils.envOverride("CONTAINER_REGISTRY_DOMAIN", "gcr.io"),
   appDistributionOrigin: utils.envOverride(
     "FIREBASE_APP_DISTRIBUTION_URL",
     "https://firebaseappdistribution.googleapis.com"

src/deploy/functions/containerCleaner.ts

@@ -0,0 +1,209 @@
+// This code is very aggressive about running requests in parallel and does not use
+// a task queue, because the quota limits for GCR.io are absurdly high. At the time
+// of writing, we can make 50K requests per 10m.
+// https://cloud.google.com/container-registry/quotas
+
+import * as clc from "cli-color";
+
+import { containerRegistryDomain } from "../../api";
+import { logger } from "../../logger";
+import * as docker from "../../gcp/docker";
+import * as backend from "./backend";
+import * as utils from "../../utils";
+
+// A flattening of container_registry_hosts and
+// region_multiregion_map from regionconfig.borg
+const SUBDOMAIN_MAPPING: Record<string, string> = {
+  "us-west2": "us",
+  "us-west3": "us",
+  "us-west4": "us",
+  "us-central1": "us",
+  "us-central2": "us",
+  "us-east1": "us",
+  "us-east4": "us",
+  "northamerica-northeast1": "us",
+  "southamerica-east1": "us",
+  "europe-west1": "eu",
+  "europe-west2": "eu",
+  "europe-west3": "eu",
+  "europe-west5": "eu",
+  "europe-west6": "eu",
+  "europe-central2": "eu",
+  "asia-east1": "asia",
+  "asia-east2": "asia",
+  "asia-northeast1": "asia",
+  "asia-northeast2": "asia",
+  "asia-northeast3": "asia",
+  "asia-south1": "asia",
+  "asia-southeast2": "asia",
+  "australia-southeast1": "asia",
+};
+
+export async function cleanupBuildImages(functions: backend.FunctionSpec[]): Promise<void> {
+  utils.logBullet(clc.bold.cyan("functions: ") + "cleaning up build files...");
+  const gcrCleaner = new ContainerRegistryCleaner();
+  const failedDomains: Set<string> = new Set();
+  await Promise.all(
+    functions.map((func) =>
+      (async () => {
+        try {
+          await gcrCleaner.cleanupFunction(func);
+        } catch (err) {
+          const path = `${func.project}/${SUBDOMAIN_MAPPING[func.region]}/gcf`;
+          failedDomains.add(`https://console.cloud.google.com/gcr/images/${path}`);
+        }
+      })()
+    )
+  );
+  if (failedDomains.size) {
+    let message =
+      "Unhandled error cleaning up build images. This could result in a small monthly bill if not corrected. ";
+    message +=
+      "You can attempt to delete these images by redeploying or you can delete them manually at";
+    if (failedDomains.size == 1) {
+      message += " " + failedDomains.values().next().value;
+    } else {
+      message += [...failedDomains].map((domain) => "\n\t" + domain).join("");
+    }
+    utils.logLabeledWarning("functions", message);
+  }
+
+  // TODO: clean up Artifact Registry images as well.
+}
+
+export class ContainerRegistryCleaner {
+  readonly helpers: Record<string, DockerHelper> = {};
+
+  private helper(location: string): DockerHelper {
+    const subdomain = SUBDOMAIN_MAPPING[location] || "us";
+    if (!this.helpers[subdomain]) {
+      const origin = `https://${subdomain}.${containerRegistryDomain}`;
+      this.helpers[subdomain] = new DockerHelper(origin);
+    }
+    return this.helpers[subdomain];
+  }
+
+  // GCFv1 has the directory structure:
+  // gcf/
+  //  +- <region>/
+  //        +- <uuid>
+  //             +- <hash> (tags: <FuncName>_version-<#>)
+  //             +- cache/ (Only present in first deploy of region)
+  //             |    +- <hash> (tags: latest)
+  //             +- worker/ (Only present in first deploy of region)
+  //                  +- <hash> (tags: latest)
+  //
+  // We'll parallel search for the valid <uuid> and their children
+  // until we find one with the right tag for the function name.
+  // The underlying Helper's caching should make this expensive for
+  // the first function and free for the next functions in the same
+  // region.
+  async cleanupFunction(func: backend.FunctionSpec): Promise<void> {
+    const helper = this.helper(func.region);
+    const uuids = (await helper.ls(`${func.project}/gcf/${func.region}`)).children;
+
+    const uuidTags: Record<string, string[]> = {};
+    const loadUuidTags: Promise<void>[] = [];
+    for (const uuid of uuids) {
+      loadUuidTags.push(
+        (async () => {
+          const path = `${func.project}/gcf/${func.region}/${uuid}`;
+          const tags = (await helper.ls(path)).tags;
+          uuidTags[path] = tags;
+        })()
+      );
+    }
+    await Promise.all(loadUuidTags);
+
+    const extractFunction = /^(.*)_version-\d+$/;
+    const entry = Object.entries(uuidTags).find(([, tags]) => {
+      return tags.find((tag) => extractFunction.exec(tag)?.[1] === func.id);
+    });
+
+    if (!entry) {
+      logger.debug("Could not find image for function", backend.functionName(func));
+      return;
+    }
+    await helper.rm(entry[0]);
+  }
+}
+
+export interface Stat {
+  children: string[];
+  digests: docker.Digest[];
+  tags: docker.Tag[];
+}
+
+export class DockerHelper {
+  readonly client: docker.Client;
+  readonly cache: Record<string, Stat> = {};
+
+  constructor(origin: string) {
+    this.client = new docker.Client(origin);
+  }
+
+  async ls(path: string): Promise<Stat> {
+    if (!this.cache[path]) {
+      const raw = await this.client.listTags(path);
+      this.cache[path] = {
+        tags: raw.tags,
+        digests: Object.keys(raw.manifest),
+        children: raw.child,
+      };
+    }
+    return this.cache[path];
+  }
+
+  // While we can't guarantee all promises will succeed, we can do our darndest
+  // to expunge as much as possible before throwing.
+  async rm(path: string): Promise<void> {
+    let toThrowLater: any = undefined;
+    const stat = await this.ls(path);
+    const recursive = stat.children.map((child) =>
+      (async () => {
+        try {
+          await this.rm(`${path}/${child}`);
+          stat.children.splice(stat.children.indexOf(child), 1);
+        } catch (err) {
+          toThrowLater = err;
+        }
+      })()
+    );
+    // Unlike a filesystem, we can delete a "directory" while its children are still being
+    // deleted. Run these in parallel to improve performance and just wait for the result
+    // before the function's end.
+
+    // An image cannot be deleted until its tags have been removed. Do this in two phases.
+    const deleteTags = stat.tags.map((tag) =>
+      (async () => {
+        try {
+          await this.client.deleteTag(path, tag);
+          stat.tags.splice(stat.tags.indexOf(tag), 1);
+        } catch (err) {
+          logger.debug("Got error trying to remove docker tag:", err);
+          toThrowLater = err;
+        }
+      })()
+    );
+    await Promise.all(deleteTags);
+
+    const deleteImages = stat.digests.map((digest) =>
+      (async () => {
+        try {
+          await this.client.deleteImage(path, digest);
+          stat.digests.splice(stat.digests.indexOf(digest), 1);
+        } catch (err) {
+          logger.debug("Got error trying to remove docker image:", err);
+          toThrowLater = err;
+        }
+      })()
+    );
+    await Promise.all(deleteImages);
+
+    await Promise.all(recursive);
+
+    if (toThrowLater) {
+      throw toThrowLater;
+    }
+  }
+}

src/deploy/functions/release.ts

@@ -7,12 +7,13 @@ import { getAppEngineLocation } from "../../functionsConfig";
 import { promptForFunctionDeletion } from "./prompts";
 import { DeploymentTimer } from "./deploymentTimer";
 import { ErrorHandler } from "./errorHandler";
-import * as utils from "../../utils";
+import { Options } from "../../options";
+import * as args from "./args";
+import * as backend from "./backend";
+import * as containerCleaner from "./containerCleaner";
 import * as helper from "./functionsDeployHelper";
 import * as tasks from "./tasks";
-import * as backend from "./backend";
-import * as args from "./args";
-import { Options } from "../../options";
+import * as utils from "../../utils";
 
 export async function release(context: args.Context, options: Options, payload: args.Payload) {
   if (!options.config.has("functions")) {
@@ -133,6 +134,7 @@ export async function release(context: args.Context, options: Options, payload:
     );
   }
   helper.logAndTrackDeployStats(cloudFunctionsQueue, errorHandler);
+  await containerCleaner.cleanupBuildImages(payload.functions!.backend.cloudFunctions);
   await helper.printTriggerUrls(context);
   errorHandler.printWarnings();
   errorHandler.printErrors();

src/gcp/docker.ts

@@ -0,0 +1,88 @@
+// Note: unlike Google APIs, the documentation for the GCR API is
+// actually the Docker REST API. This can be found at
+// https://docs.docker.com/registry/spec/api/
+// This API is _very_ complex in its entirety and is very subtle (e.g. tags and digests
+// are both strings and can both be put in the same route to get completely different
+// response document types).
+// This file will only implement a minimal subset as needed.
+import { FirebaseError } from "../error";
+import * as api from "../apiv2";
+
+// A Digest is a string in the format <algorithm>:<hex>. For example:
+// sha256:146d8c9dff0344fb01417ef28673ed196e38215f3c94837ae733d3b064ba439e
+export type Digest = string;
+export type Tag = string;
+
+export interface Tags {
+  name: string;
+  tags: string[];
+
+  // These fields are not documented in the Docker API but are
+  // present in the GCR API.
+  manifest: Record<Digest, ImageInfo>;
+  child: string[];
+}
+
+export interface ImageInfo {
+  // times are string milliseconds
+  timeCreatedMs: string;
+  timeUploadedMs: string;
+  tag: string[];
+  mediaType: string;
+  imageSizeBytes: string;
+  layerId: string;
+}
+
+interface ErrorsResponse {
+  errors?: {
+    code: string;
+    message: string;
+    details: unknown;
+  }[];
+}
+
+function isErrors(response: unknown): response is ErrorsResponse {
+  return Object.prototype.hasOwnProperty.call(response, "errors");
+}
+
+const API_VERSION = "v2";
+
+export class Client {
+  readonly client: api.Client;
+
+  constructor(origin: string) {
+    this.client = new api.Client({
+      apiVersion: API_VERSION,
+      auth: true,
+      urlPrefix: origin,
+    });
+  }
+
+  async listTags(path: string): Promise<Tags> {
+    const response = await this.client.get<Tags | ErrorsResponse>(`${path}/tags/list`);
+    if (isErrors(response.body)) {
+      throw new FirebaseError(`Failed to list GCR tags at ${path}`, {
+        children: response.body.errors,
+      });
+    }
+    return response.body;
+  }
+
+  async deleteTag(path: string, tag: Tag): Promise<void> {
+    const response = await this.client.delete<ErrorsResponse>(`${path}/manifests/${tag}`);
+    if (response.body.errors?.length != 0) {
+      throw new FirebaseError(`Failed to delete tag ${tag} at path ${path}`, {
+        children: response.body.errors,
+      });
+    }
+  }
+
+  async deleteImage(path: string, digest: Digest): Promise<void> {
+    const response = await this.client.delete<ErrorsResponse>(`${path}/manifests/${digest}`);
+    if (response.body.errors?.length != 0) {
+      throw new FirebaseError(`Failed to delete image ${digest} at path ${path}`, {
+        children: response.body.errors,
+      });
+    }
+  }
+}