Enhance the IAM role binding process (#4511)

* change up ordering, add jump out points, and add sleep

* linting

* yanking out sleep, adding in a better error message for the first deploy, and replacing reduce with our functional module

* adding changelog

CHANGELOG.md

@@ -1,2 +1,3 @@
 - Add support for secrets to v2 functions (#4451).
 - Fixes an issue where `ext:export` would write different param values than what it displayed in the prompt (#4515).
+- Enhances the functions IAM permission process and updates the error messages (#4511).

src/deploy/functions/checkIam.ts

@@ -115,6 +115,16 @@ export async function checkHttpIam(
   logger.debug("[functions] found setIamPolicy permission, proceeding with deploy");
 }
 
+/** obtain the pubsub service agent */
+function getPubsubServiceAgent(projectNumber: string): string {
+  return `serviceAccount:service-${projectNumber}@gcp-sa-pubsub.iam.gserviceaccount.com`;
+}
+
+/** obtain the default compute service agent */
+function getDefaultComputeServiceAgent(projectNumber: string): string {
+  return `serviceAccount:${projectNumber}-compute@developer.gserviceaccount.com`;
+}
+
 /** Callback reducer function */
 function reduceEventsToServices(services: Array<Service>, endpoint: backend.Endpoint) {
   const service = serviceForEndpoint(endpoint);
@@ -124,43 +134,18 @@ function reduceEventsToServices(services: Array<Service>, endpoint: backend.Endp
   return services;
 }
 
-/**
- * Returns the IAM bindings that grants the role to the service account
- * @param existingPolicy the project level IAM policy
- * @param serviceAccount the IAM service account
- * @param role the role you want to grant
- * @return the correct IAM binding
- */
-export function obtainBinding(
-  existingPolicy: iam.Policy,
-  serviceAccount: string,
-  role: string
-): iam.Binding {
-  let binding = existingPolicy.bindings.find((b) => b.role === role);
-  if (!binding) {
-    binding = {
-      role,
-      members: [],
-    };
-  }
-  if (!binding.members.find((m) => m === serviceAccount)) {
-    binding.members.push(serviceAccount);
-  }
-  return binding;
-}
-
 /**
  * Finds the required project level IAM bindings for the Pub/Sub service agent.
  * If the user enabled Pub/Sub on or before April 8, 2021, then we must enable the token creator role.
  * @param projectNumber project number
  * @param existingPolicy the project level IAM policy
  */
-export function obtainPubSubServiceAgentBindings(
-  projectNumber: string,
-  existingPolicy: iam.Policy
-): iam.Binding[] {
-  const pubsubServiceAgent = `serviceAccount:service-${projectNumber}@gcp-sa-pubsub.iam.gserviceaccount.com`;
-  return [obtainBinding(existingPolicy, pubsubServiceAgent, SERVICE_ACCOUNT_TOKEN_CREATOR_ROLE)];
+export function obtainPubSubServiceAgentBindings(projectNumber: string): iam.Binding[] {
+  const serviceAccountTokenCreatorBinding: iam.Binding = {
+    role: SERVICE_ACCOUNT_TOKEN_CREATOR_ROLE,
+    members: [getPubsubServiceAgent(projectNumber)],
+  };
+  return [serviceAccountTokenCreatorBinding];
 }
 
 /**
@@ -169,54 +154,73 @@ export function obtainPubSubServiceAgentBindings(
  * @param projectNumber project number
  * @param existingPolicy the project level IAM policy
  */
-export function obtainDefaultComputeServiceAgentBindings(
-  projectNumber: string,
-  existingPolicy: iam.Policy
-): iam.Binding[] {
-  const defaultComputeServiceAgent = `serviceAccount:${projectNumber}-compute@developer.gserviceaccount.com`;
-  const invokerBinding = obtainBinding(
-    existingPolicy,
-    defaultComputeServiceAgent,
-    RUN_INVOKER_ROLE
-  );
-  const eventReceiverBinding = obtainBinding(
-    existingPolicy,
-    defaultComputeServiceAgent,
-    EVENTARC_EVENT_RECEIVER_ROLE
-  );
-  return [invokerBinding, eventReceiverBinding];
+export function obtainDefaultComputeServiceAgentBindings(projectNumber: string): iam.Binding[] {
+  const defaultComputeServiceAgent = getDefaultComputeServiceAgent(projectNumber);
+  const runInvokerBinding: iam.Binding = {
+    role: RUN_INVOKER_ROLE,
+    members: [defaultComputeServiceAgent],
+  };
+  const eventarcEventReceiverBinding: iam.Binding = {
+    role: EVENTARC_EVENT_RECEIVER_ROLE,
+    members: [defaultComputeServiceAgent],
+  };
+  return [runInvokerBinding, eventarcEventReceiverBinding];
 }
 
-/** Helper to merge all required bindings into the IAM policy */
-export function mergeBindings(policy: iam.Policy, allRequiredBindings: iam.Binding[][]) {
-  for (const requiredBindings of allRequiredBindings) {
-    if (requiredBindings.length === 0) {
+/** Helper to merge all required bindings into the IAM policy, returns boolean if the policy has been updated */
+export function mergeBindings(policy: iam.Policy, requiredBindings: iam.Binding[]): boolean {
+  let updated = false;
+  for (const requiredBinding of requiredBindings) {
+    const match = policy.bindings.find((b) => b.role === requiredBinding.role);
+    if (!match) {
+      updated = true;
+      policy.bindings.push(requiredBinding);
       continue;
     }
-    for (const requiredBinding of requiredBindings) {
-      const ndx = policy.bindings.findIndex(
-        (policyBinding) => policyBinding.role === requiredBinding.role
-      );
-      if (ndx === -1) {
-        policy.bindings.push(requiredBinding);
-        continue;
+    for (const requiredMember of requiredBinding.members) {
+      if (!match.members.find((m) => m === requiredMember)) {
+        updated = true;
+        match.members.push(requiredMember);
       }
-      requiredBinding.members.forEach((updatedMember) => {
-        if (!policy.bindings[ndx].members.find((member) => member === updatedMember)) {
-          policy.bindings[ndx].members.push(updatedMember);
-        }
-      });
+    }
+  }
+  return updated;
+}
+
+/** Utility to print the required binding commands */
+function printManualIamConfig(requiredBindings: iam.Binding[], projectId: string) {
+  utils.logLabeledBullet(
+    "functions",
+    "Failed to verify the project has the correct IAM bindings for a successful deployment.",
+    "warn"
+  );
+  utils.logLabeledBullet(
+    "functions",
+    "You can either re-run `firebase deploy` as a project owner or manually run the following set of `gcloud` commands:",
+    "warn"
+  );
+  for (const binding of requiredBindings) {
+    for (const member of binding.members) {
+      utils.logLabeledBullet(
+        "functions",
+        `\`gcloud projects add-iam-policy-binding ${projectId} ` +
+          `--member=${member} ` +
+          `--role=${binding.role}\``,
+        "warn"
+      );
     }
   }
 }
 
 /**
  * Checks and sets the roles for specific resource service agents
+ * @param projectId human readable project id
  * @param projectNumber project number
  * @param want backend that we want to deploy
  * @param have backend that we have currently deployed
  */
 export async function ensureServiceAgentRoles(
+  projectId: string,
   projectNumber: string,
   want: backend.Backend,
   have: backend.Backend
@@ -230,11 +234,28 @@ export async function ensureServiceAgentRoles(
   if (newServices.length === 0) {
     return;
   }
+
+  // obtain all the bindings we need to have active in the project
+  const requiredBindingsPromises: Array<Promise<Array<iam.Binding>>> = [];
+  for (const service of newServices) {
+    requiredBindingsPromises.push(service.requiredProjectBindings!(projectNumber));
+  }
+  const nestedRequiredBindings = await Promise.all(requiredBindingsPromises);
+  const requiredBindings = [...flattenArray(nestedRequiredBindings)];
+  if (haveServices.length === 0) {
+    requiredBindings.push(...obtainPubSubServiceAgentBindings(projectNumber));
+    requiredBindings.push(...obtainDefaultComputeServiceAgentBindings(projectNumber));
+  }
+  if (requiredBindings.length === 0) {
+    return;
+  }
+
   // get the full project iam policy
   let policy: iam.Policy;
   try {
     policy = await getIamPolicy(projectNumber);
   } catch (err: any) {
+    printManualIamConfig(requiredBindings, projectId);
     utils.logLabeledBullet(
       "functions",
       "Could not verify the necessary IAM configuration for the following newly-integrated services: " +
@@ -244,24 +265,16 @@ export async function ensureServiceAgentRoles(
     );
     return;
   }
-  // run in parallel all the missingProjectBindings jobs
-  const findRequiredBindings: Array<Promise<Array<iam.Binding>>> = [];
-  newServices.forEach((service) =>
-    findRequiredBindings.push(service.requiredProjectBindings!(projectNumber, policy))
-  );
-  const allRequiredBindings = await Promise.all(findRequiredBindings);
-  if (haveServices.length === 0) {
-    allRequiredBindings.push(obtainPubSubServiceAgentBindings(projectNumber, policy));
-    allRequiredBindings.push(obtainDefaultComputeServiceAgentBindings(projectNumber, policy));
-  }
-  if (!allRequiredBindings.find((bindings) => bindings.length > 0)) {
+  const hasUpdatedBindings = mergeBindings(policy, requiredBindings);
+  if (!hasUpdatedBindings) {
     return;
   }
-  mergeBindings(policy, allRequiredBindings);
+
   // set the updated policy
   try {
     await setIamPolicy(projectNumber, policy, "bindings");
   } catch (err: any) {
+    printManualIamConfig(requiredBindings, projectId);
     throw new FirebaseError(
       "We failed to modify the IAM policy for the project. The functions " +
         "deployment requires specific roles to be granted to service agents," +

src/deploy/functions/prepare.ts

@@ -28,6 +28,7 @@ import { FirebaseError } from "../../error";
 import { configForCodebase, normalizeAndValidate } from "../../functions/projectConfig";
 import { previews } from "../../previews";
 import { AUTH_BLOCKING_EVENTS } from "../../functions/events/v1";
+import { generateServiceIdentity } from "../../gcp/serviceusage";
 
 function hasUserConfig(config: Record<string, unknown>): boolean {
   // "firebase" key is always going to exist in runtime config.
@@ -209,10 +210,10 @@ export async function prepare(
       return ensureApiEnabled.ensure(projectId, api, "functions", /* silent=*/ false);
     })
   );
-  // Note: Some of these are premium APIs that require billing to be enabled.
-  // We'd eventually have to add special error handling for billing APIs, but
-  // enableCloudBuild is called above and has this special casing already.
   if (backend.someEndpoint(wantBackend, (e) => e.platform === "gcfv2")) {
+    // Note: Some of these are premium APIs that require billing to be enabled.
+    // We'd eventually have to add special error handling for billing APIs, but
+    // enableCloudBuild is called above and has this special casing already.
     const V2_APIS = [
       "artifactregistry.googleapis.com",
       "run.googleapis.com",
@@ -224,6 +225,13 @@ export async function prepare(
       return ensureApiEnabled.ensure(context.projectId, api, "functions");
     });
     await Promise.all(enablements);
+    // Need to manually kick off the p4sa activation of services
+    // that we use with IAM roles assignment.
+    const services = ["pubsub.googleapis.com", "eventarc.googleapis.com"];
+    const generateServiceAccounts = services.map((service) => {
+      return generateServiceIdentity(projectNumber, service, "functions");
+    });
+    await Promise.all(generateServiceAccounts);
   }
 
   // ===Phase 5. Ask for user prompts for things might warrant user attentions.
@@ -237,7 +245,7 @@ export async function prepare(
   // ===Phase 6. Finalize preparation by "fixing" all extraneous environment issues like IAM policies.
   // We limit the scope endpoints being deployed.
   await backend.checkAvailability(context, matchingBackend);
-  await ensureServiceAgentRoles(projectNumber, matchingBackend, haveBackend);
+  await ensureServiceAgentRoles(projectId, projectNumber, matchingBackend, haveBackend);
   await validate.secretsAreValid(projectId, matchingBackend);
   await ensure.secretAccess(projectId, matchingBackend, haveBackend);
 }

src/deploy/functions/services/index.ts

@@ -20,10 +20,7 @@ export interface Service {
   readonly api: string;
 
   // dispatch functions
-  requiredProjectBindings?: (
-    projectNumber: string,
-    policy: iam.Policy
-  ) => Promise<Array<iam.Binding>>;
+  requiredProjectBindings?: (projectNumber: string) => Promise<Array<iam.Binding>>;
   ensureTriggerRegion: (ep: backend.Endpoint & backend.EventTriggered) => Promise<void>;
   validateTrigger: (ep: backend.Endpoint, want: backend.Backend) => void;
   registerTrigger: (ep: backend.Endpoint) => Promise<void>;

src/deploy/functions/services/storage.ts

@@ -12,23 +12,15 @@ const PUBSUB_PUBLISHER_ROLE = "roles/pubsub.publisher";
  * @param projectId project identifier
  * @param existingPolicy the project level IAM policy
  */
-export async function obtainStorageBindings(
-  projectNumber: string,
-  existingPolicy: iam.Policy
-): Promise<Array<iam.Binding>> {
+export async function obtainStorageBindings(projectNumber: string): Promise<Array<iam.Binding>> {
   const storageResponse = await storage.getServiceAccount(projectNumber);
   const storageServiceAgent = `serviceAccount:${storageResponse.email_address}`;
-  let pubsubBinding = existingPolicy.bindings.find((b) => b.role === PUBSUB_PUBLISHER_ROLE);
-  if (!pubsubBinding) {
-    pubsubBinding = {
-      role: PUBSUB_PUBLISHER_ROLE,
-      members: [],
-    };
-  }
-  if (!pubsubBinding.members.find((m) => m === storageServiceAgent)) {
-    pubsubBinding.members.push(storageServiceAgent); // add service agent to role
-  }
-  return [pubsubBinding];
+
+  const pubsubPublisherBinding = {
+    role: PUBSUB_PUBLISHER_ROLE,
+    members: [storageServiceAgent],
+  };
+  return [pubsubPublisherBinding];
 }
 
 /**

src/gcp/cloudfunctionsv2.ts

@@ -240,12 +240,23 @@ export function mebibytes(memory: string): number {
  * @param err The error returned from the operation.
  */
 function functionsOpLogReject(funcName: string, type: string, err: any): void {
+  utils.logWarning(clc.bold.yellow("functions:") + ` ${err?.message}`);
   if (err?.context?.response?.statusCode === 429) {
     utils.logWarning(
       `${clc.bold.yellow(
         "functions:"
       )} got "Quota Exceeded" error while trying to ${type} ${funcName}. Waiting to retry...`
     );
+  } else if (
+    err?.message.includes(
+      "If you recently started to use Eventarc, it may take a few minutes before all necessary permissions are propagated to the Service Agent"
+    )
+  ) {
+    utils.logWarning(
+      `${clc.bold.yellow(
+        "functions:"
+      )} since this is your first time using functions v2, we need a little bit longer to finish setting everything up, please retry the deployment in a few minutes.`
+    );
   } else {
     utils.logWarning(
       clc.bold.yellow("functions:") + " failed to " + type + " function " + funcName

src/gcp/serviceusage.ts

@@ -0,0 +1,33 @@
+import { bold } from "cli-color";
+import { serviceUsageOrigin } from "../api";
+import { Client } from "../apiv2";
+import { FirebaseError } from "../error";
+import * as utils from "../utils";
+
+const apiClient = new Client({
+  urlPrefix: serviceUsageOrigin,
+  apiVersion: "v1beta1",
+});
+
+/**
+ * Generate the service account for the service. Note: not every service uses the endpoint.
+ * @param projectNumber gcp project number
+ * @param service the service api (ex~ pubsub.googleapis.com)
+ * @returns
+ */
+export async function generateServiceIdentity(
+  projectNumber: string,
+  service: string,
+  prefix: string
+) {
+  utils.logLabeledBullet(prefix, `generating the service identity for ${bold(service)}...`);
+  try {
+    return await apiClient.post<unknown, unknown>(
+      `projects/${projectNumber}/services/${service}:generateServiceIdentity`
+    );
+  } catch (err: unknown) {
+    throw new FirebaseError(`Error generating the service identity for ${service}.`, {
+      original: err as Error,
+    });
+  }
+}