Delete Container Registry images left after Functions deployment

src/api.js

@@ -99,6 +99,7 @@ var api = {
     "FIREBASE_CLOUDLOGGING_URL",
     "https://logging.googleapis.com"
   ),
+  containerRegistryDomain: utils.envOverride("CONTAINER_REGISTRY_DOMAIN", "gcr.io"),
   appDistributionOrigin: utils.envOverride(
     "FIREBASE_APP_DISTRIBUTION_URL",
     "https://firebaseappdistribution.googleapis.com"

src/deploy/functions/containerCleaner.ts

@@ -0,0 +1,207 @@
+// This code is very aggressive about running requests in parallel and does not use
+// a task queue, because the quota limits for GCR.io are absurdly high. At the time
+// of writing, we can make 50K requests per 10m.
+// https://cloud.google.com/container-registry/quotas
+
+import * as clc from "cli-color";
+
+import { logger } from "../../logger";
+import * as gcr from "../../gcp/containerregistry";
+import * as backend from "./backend";
+import * as utils from "../../utils";
+
+// A flattening of container_registry_hosts and
+// region_multiregion_map from regionconfig.borg
+const SUBDOMAIN_MAPPING: Record<string, string> = {
+  "us-west2": "us",
+  "us-west3": "us",
+  "us-west4": "us",
+  "us-central1": "us",
+  "us-central2": "us",
+  "us-east1": "us",
+  "us-east4": "us",
+  "northamerica-northeast1": "us",
+  "southamerica-east1": "us",
+  "europe-west1": "eu",
+  "europe-west2": "eu",
+  "europe-west3": "eu",
+  "europe-west5": "eu",
+  "europe-west6": "eu",
+  "europe-central2": "eu",
+  "asia-east1": "asia",
+  "asia-east2": "asia",
+  "asia-northeast1": "asia",
+  "asia-northeast2": "asia",
+  "asia-northeast3": "asia",
+  "asia-south1": "asia",
+  "asia-southeast2": "asia",
+  "australia-southeast1": "asia",
+};
+
+export async function cleanupBuildImages(functions: backend.FunctionSpec[]): Promise<void> {
+  utils.logBullet(clc.bold.cyan("functions: ") + "cleaning up build files...");
+  const gcrCleaner = new ContainerRegistryCleaner();
+  try {
+    await Promise.all(functions.map((func) => gcrCleaner.cleanupFunction(func)));
+  } catch (err) {
+    logger.debug("Failed to delete container registry artifacts with error", err);
+    utils.logLabeledWarning(
+      "functions",
+      "Unhnandled error cleaning up build files. This could result in a small monthly bill if not corrected"
+    );
+  }
+
+  // TODO: clean up Artifact Registry images as well.
+}
+
+export class ContainerRegistryCleaner {
+  readonly helpers: Record<string, ContainerRegistryHelper> = {};
+
+  private helper(location: string): ContainerRegistryHelper {
+    const subdomain = SUBDOMAIN_MAPPING[location] || "us";
+    if (!this.helpers[subdomain]) {
+      this.helpers[subdomain] = new ContainerRegistryHelper(subdomain);
+    }
+    return this.helpers[subdomain];
+  }
+
+  // GCFv1 has the directory structure:
+  // gcf/
+  //  +- <region>/
+  //        +- <uuid>
+  //             +- <hash> (tags: <FuncName>_version-<#>)
+  //             +- cache/ (Only present in first deploy of region)
+  //             |    +- <hash> (tags: latest)
+  //             +- worker/ (Only present in first deploy of region)
+  //                  +- <hash> (tags: latest)
+  //
+  // We'll parallel search for the valid <uuid> and their children
+  // until we find one with the right tag for the function name.
+  // The underlying Helper's caching should make this expensive for
+  // the first function and free for the next functions in the same
+  // region.
+  async cleanupFunction(func: backend.FunctionSpec): Promise<void> {
+    const helper = this.helper(func.region);
+    const uuids = (await helper.ls(`${func.project}/gcf/${func.region}`)).children;
+
+    const uuidTags: Record<string, string[]> = {};
+    const loadUuidTags: Promise<void>[] = [];
+    for (const uuid of uuids) {
+      loadUuidTags.push(
+        (async () => {
+          const path = `${func.project}/gcf/${func.region}/${uuid}`;
+          const tags = (await helper.ls(path)).tags;
+          uuidTags[path] = tags;
+        })()
+      );
+    }
+    await Promise.all(loadUuidTags);
+
+    const extractFunction = /^(.*)_version-\d+$/;
+    const entry = Object.entries(uuidTags).find(([, tags]) => {
+      return tags.find((tag) => {
+        const match = tag.match(extractFunction);
+        return match && match[1] === func.id;
+      });
+    });
+
+    if (!entry) {
+      logger.debug("Could not find image for function", backend.functionName(func));
+      return;
+    }
+    await helper.rm(entry[0]);
+  }
+}
+
+export interface Stat {
+  children: string[];
+  digests: gcr.Digest[];
+  tags: gcr.Tag[];
+}
+
+export interface Node {
+  // If we haven't actually done an LS on this exact location
+  sparse: boolean;
+
+  // For directories
+  children: Record<string, Node>;
+
+  // For images
+  digests: gcr.Digest[];
+  tags: gcr.Tag[];
+}
+
+export class ContainerRegistryHelper {
+  readonly client: gcr.Client;
+  readonly cache: Node;
+
+  constructor(subdomain: string) {
+    this.client = new gcr.Client(subdomain);
+    this.cache = {
+      sparse: true,
+      children: {},
+      digests: [],
+      tags: [],
+    };
+  }
+
+  private async getNode(path: string): Promise<Node> {
+    const parts = path.split("/");
+    let cwd = this.cache;
+    for (const part of parts) {
+      if (!cwd.children[part]) {
+        cwd.children[part] = {
+          sparse: true,
+          children: {},
+          digests: [],
+          tags: [],
+        };
+      }
+      cwd = cwd.children[part];
+    }
+    if (cwd.sparse) {
+      const raw = await this.client.listTags(path);
+      cwd.sparse = false;
+      cwd.tags = raw.tags;
+      cwd.digests = Object.keys(raw.manifest);
+      cwd.children = {};
+      for (const child of raw.child) {
+        cwd.children[child] = {
+          sparse: true,
+          children: {},
+          digests: [],
+          tags: [],
+        };
+      }
+    }
+    return cwd;
+  }
+
+  async ls(path: string): Promise<Stat> {
+    const node = await this.getNode(path);
+    return {
+      children: Object.keys(node.children),
+      digests: node.digests,
+      tags: node.tags,
+    };
+  }
+
+  async rm(path: string): Promise<void> {
+    const node = await this.getNode(path);
+    const deleteChildren: Promise<void>[] = [];
+    const recursive = Object.keys(node.children).map((child) => this.rm(`${path}/${child}`));
+    // Let children ("directories") be cleaned up in parallel while we clean
+    // up the "files" in this location.
+
+    const deleteTags = node.tags.map((tag) => this.client.deleteTag(path, tag));
+    await Promise.all(deleteTags);
+    node.tags = [];
+
+    const deleteImages = node.digests.map((digest) => this.client.deleteImage(path, digest));
+    await Promise.all(deleteImages);
+    node.digests = [];
+
+    await Promise.all(recursive);
+    node.children = {};
+  }
+}

src/deploy/functions/release.ts

@@ -7,12 +7,13 @@ import { getAppEngineLocation } from "../../functionsConfig";
 import { promptForFunctionDeletion } from "./prompts";
 import { DeploymentTimer } from "./deploymentTimer";
 import { ErrorHandler } from "./errorHandler";
-import * as utils from "../../utils";
+import { Options } from "../../options";
+import * as args from "./args";
+import * as backend from "./backend";
+import * as containerCleaner from "./containerCleaner";
 import * as helper from "./functionsDeployHelper";
 import * as tasks from "./tasks";
-import * as backend from "./backend";
-import * as args from "./args";
-import { Options } from "../../options";
+import * as utils from "../../utils";
 
 export async function release(context: args.Context, options: Options, payload: args.Payload) {
   if (!options.config.has("functions")) {
@@ -136,6 +137,7 @@ export async function release(context: args.Context, options: Options, payload:
     );
   }
   helper.logAndTrackDeployStats(cloudFunctionsQueue, errorHandler);
+  await containerCleaner.cleanupBuildImages(payload.functions!.backend.cloudFunctions);
   await helper.printTriggerUrls(context);
   errorHandler.printWarnings();
   errorHandler.printErrors();

src/gcp/containerregistry.ts

@@ -0,0 +1,95 @@
+// Note: unlike Google APIs, the documentation for the GCR API is
+// actually the Docker REST API. This can be found at
+// https://docs.docker.com/registry/spec/api/
+// This API is _very_ complex in its entirety and is very subtle (e.g. tags and digests
+// are both strings and can both be put in the same route to get completely different
+// response document types).
+// This file will only implement a minimal subset as needed.
+import { FirebaseError } from "../error";
+import { containerRegistryDomain } from "../api";
+import * as api from "../apiv2";
+
+// A Digest is a string in the format <algorithm>:<hex>. For example:
+// sha256:146d8c9dff0344fb01417ef28673ed196e38215f3c94837ae733d3b064ba439e
+export type Digest = string;
+export type Tag = string;
+
+export interface Tags {
+  name: string;
+  tags: string[];
+
+  // These fields are not documented in the Docker API but are
+  // present in the GCR API.
+  manifest: Record<Digest, ImageInfo>;
+  child: string[];
+}
+
+export interface ImageInfo {
+  // times are string milliseconds
+  timeCreatedMs: string;
+  timeUploadedMs: string;
+  tag: string[];
+  mediaType: string;
+  imageSizeBytes: string;
+  layerId: string;
+}
+
+interface ErrorsResponse {
+  errors?: {
+    code: string;
+    message: string;
+    details: unknown;
+  }[];
+}
+
+function isErrors(response: unknown): response is ErrorsResponse {
+  return Object.prototype.hasOwnProperty.bind(response)("errors");
+}
+
+const API_VERSION = "v2";
+
+export class Client {
+  readonly client: api.Client;
+
+  constructor(subdomain?: string) {
+    let origin: string;
+    if (subdomain) {
+      origin = `https://${subdomain}.${containerRegistryDomain}`;
+    } else {
+      origin = `https://${containerRegistryDomain}`;
+    }
+    this.client = new api.Client({
+      apiVersion: API_VERSION,
+      auth: true,
+      urlPrefix: origin,
+    });
+  }
+
+  async listTags(path: string): Promise<Tags> {
+    const response = await this.client.get<Tags | ErrorsResponse>(`${path}/tags/list`);
+    if (isErrors(response.body)) {
+      throw new FirebaseError(`Failed to list GCR tags at ${path}`, {
+        children: response.body.errors,
+      });
+    }
+    return response.body;
+  }
+
+  async deleteTag(path: string, tag: Tag): Promise<void> {
+    const response = await this.client.delete<ErrorsResponse>(`${path}/manifests/${tag}`);
+    if (response.body.errors?.length != 0) {
+      throw new FirebaseError(`Failed to delete tag ${tag} at path ${path}`, {
+        children: response.body.errors,
+      });
+    }
+  }
+
+  async deleteImage(path: string, digest: Digest): Promise<void> {
+    const response = await this.client.delete<ErrorsResponse>(`${path}/manifests/${digest}`);
+    if (response.body.errors?.length != 0) {
+      throw new FirebaseError(`Failed to delete image ${digest} at path ${path}`, {
+        children: response.body.errors,
+      });
+    }
+  }
+}