[7.17] [Ops] Refactor env loading & fix agent-targeting rewrite (#178320

) (#178959)

# Backport

This will backport the following commits from `main` to `7.17`:
- [[Ops] Refactor env loading & fix agent-targeting rewrite
(#178320)](#178320)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Alex
Szabo","email":"alex.szabo@elastic.co"},"sourceCommit":{"committedDate":"2024-03-12T15:31:26Z","message":"[Ops]
Refactor env loading & fix agent-targeting rewrite (#178320)\n\n##
Summary\r\nThis PR refactors a bit of the pre-command env setup,
separating parts,\r\nso they can be individually skipped. Then it
removes the setup-avoidance\r\nbased on agent types, as this won't be
useful after the migration.\r\n\r\nAlso, it fixes a missed bit in the
agent-targeting rewrite used for the\r\nmigration, where the `provider:
'gcp'` was missing, and adds an optional\r\ntargeting for the
script.\r\n\r\n- add gcp as provider to all rewritten agent targeting
rules\r\n- add option to target specific pipelines\r\n- refactor env-var
loading to a separated file\r\n- refactor node installs so it can be
switched by a flag\r\n- skip node installing in (some) jobs that don't
require
it","sha":"a89fb9b2fbb73742808fa419aaf98486c31d921d","branchLabelMapping":{"^v8.14.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Operations","release_note:skip","backport
missing","v8.14.0","v8.13.1"],"number":178320,"url":"#178320
Refactor env loading & fix agent-targeting rewrite (#178320)\n\n##
Summary\r\nThis PR refactors a bit of the pre-command env setup,
separating parts,\r\nso they can be individually skipped. Then it
removes the setup-avoidance\r\nbased on agent types, as this won't be
useful after the migration.\r\n\r\nAlso, it fixes a missed bit in the
agent-targeting rewrite used for the\r\nmigration, where the `provider:
'gcp'` was missing, and adds an optional\r\ntargeting for the
script.\r\n\r\n- add gcp as provider to all rewritten agent targeting
rules\r\n- add option to target specific pipelines\r\n- refactor env-var
loading to a separated file\r\n- refactor node installs so it can be
switched by a flag\r\n- skip node installing in (some) jobs that don't
require
it","sha":"a89fb9b2fbb73742808fa419aaf98486c31d921d"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.14.0","labelRegex":"^v8.14.0$","isSourceBranch":true,"state":"MERGED","url":"#178320
Refactor env loading & fix agent-targeting rewrite (#178320)\n\n##
Summary\r\nThis PR refactors a bit of the pre-command env setup,
separating parts,\r\nso they can be individually skipped. Then it
removes the setup-avoidance\r\nbased on agent types, as this won't be
useful after the migration.\r\n\r\nAlso, it fixes a missed bit in the
agent-targeting rewrite used for the\r\nmigration, where the `provider:
'gcp'` was missing, and adds an optional\r\ntargeting for the
script.\r\n\r\n- add gcp as provider to all rewritten agent targeting
rules\r\n- add option to target specific pipelines\r\n- refactor env-var
loading to a separated file\r\n- refactor node installs so it can be
switched by a flag\r\n- skip node installing in (some) jobs that don't
require
it","sha":"a89fb9b2fbb73742808fa419aaf98486c31d921d"}},{"branch":"8.13","label":"v8.13.1","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"url":"#178533"}]}]
BACKPORT-->

.buildkite/hooks/post-command

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-.buildkite/scripts/lifecycle/post_command.sh
+source .buildkite/scripts/lifecycle/post_command.sh

.buildkite/scripts/common/activate_service_account.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh"
+
+CALL_ARGUMENT="${1:-}"
+GCLOUD_EMAIL_POSTFIX="elastic-kibana-ci.iam.gserviceaccount.com"
+GCLOUD_SA_PROXY_EMAIL="kibana-ci-sa-proxy@$GCLOUD_EMAIL_POSTFIX"
+
+if [[ -z "$CALL_ARGUMENT" ]]; then
+  echo "Usage: $0 <bucket_name|email>"
+  exit 1
+elif [[ "$CALL_ARGUMENT" == "--unset-impersonation" ]]; then
+  echo "Unsetting impersonation"
+  gcloud config unset auth/impersonate_service_account
+  exit 0
+elif [[ "$CALL_ARGUMENT" == "--logout-gcloud" ]]; then
+  echo "Logging out of gcloud"
+  if [[ -x "$(command -v gcloud)" ]] && [[ "$(gcloud auth list 2>/dev/null | grep $GCLOUD_SA_PROXY_EMAIL)" != "" ]]; then
+    gcloud auth revoke $GCLOUD_SA_PROXY_EMAIL --no-user-output-enabled
+  fi
+  exit 0
+fi
+
+CURRENT_GCLOUD_USER=$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")
+
+# Verify that the service account proxy is activated
+if [[ "$CURRENT_GCLOUD_USER" != "$GCLOUD_SA_PROXY_EMAIL" ]]; then
+    if [[ -x "$(command -v gcloud)" ]]; then
+      if [[ -z "${KIBANA_SERVICE_ACCOUNT_PROXY_KEY:-}" ]]; then
+        echo "KIBANA_SERVICE_ACCOUNT_PROXY_KEY is not set, cannot activate service account $GCLOUD_SA_PROXY_EMAIL."
+        exit 1
+      fi
+
+      AUTH_RESULT=$(gcloud auth activate-service-account --key-file="$KIBANA_SERVICE_ACCOUNT_PROXY_KEY" || "FAILURE")
+      if [[ "$AUTH_RESULT" == "FAILURE" ]]; then
+        echo "Failed to activate service account $GCLOUD_SA_PROXY_EMAIL."
+        exit 1
+      else
+        echo "Activated service account $GCLOUD_SA_PROXY_EMAIL"
+      fi
+    else
+      echo "gcloud is not installed, cannot activate service account $GCLOUD_SA_PROXY_EMAIL."
+      exit 1
+    fi
+fi
+
+# Check if the arg is a service account e-mail or a bucket name
+EMAIL=""
+if [[ "$CALL_ARGUMENT" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ ]]; then
+  EMAIL="$CALL_ARGUMENT"
+elif [[ "$CALL_ARGUMENT" =~ ^gs://* ]]; then
+  BUCKET_NAME="${CALL_ARGUMENT:5}"
+else
+  BUCKET_NAME="$CALL_ARGUMENT"
+fi
+
+if [[ -z "$EMAIL" ]]; then
+  case "$BUCKET_NAME" in
+    "elastic-kibana-coverage-live")
+      EMAIL="kibana-ci-access-coverage@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-ci-es-snapshots-daily")
+      EMAIL="kibana-ci-access-es-daily@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-ci-es-snapshots-permanent")
+      EMAIL="kibana-ci-access-es-permanent@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-so-types-snapshots")
+      EMAIL="kibana-ci-access-so-snapshots@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-performance")
+      EMAIL="kibana-ci-access-perf-stats@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "ci-artifacts.kibana.dev")
+      EMAIL="kibana-ci-access-artifacts@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    *)
+      EMAIL="$BUCKET_NAME@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+  esac
+fi
+
+# Activate the service account
+echo "Impersonating $EMAIL"
+gcloud config set auth/impersonate_service_account "$EMAIL"
+echo "Activated service account $EMAIL"

.buildkite/scripts/common/env.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+echo '--- Setup environment vars'
+
 export CI=true
 
 KIBANA_DIR=$(pwd)

.buildkite/scripts/common/setup_buildkite_deps.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo '--- Install/build buildkite dependencies'
+
+cd '.buildkite'
+retry 5 15 npm ci
+cd -

.buildkite/scripts/common/setup_job_env.sh

@@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo '--- Job Environment Setup'
+
+if [[ "$(type -t vault_get)" != "function" ]]; then
+  source .buildkite/scripts/common/vault_fns.sh
+fi
+
+# Set up general-purpose tokens and credentials
+{
+  BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)"
+  export BUILDKITE_TOKEN
+
+  GITHUB_TOKEN=$(vault_get kibanamachine github_token)
+  export GITHUB_TOKEN
+
+  KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token)
+  export KIBANA_CI_GITHUB_TOKEN
+
+  KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
+  export KIBANA_DOCKER_USERNAME
+
+  KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
+  export KIBANA_DOCKER_PASSWORD
+}
+
+# Set up a custom ES Snapshot Manifest if one has been specified for this build
+{
+  ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')}
+  export ES_SNAPSHOT_MANIFEST
+
+  if [[ "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+  cat << EOF | buildkite-agent annotate --style "info" --context es-snapshot-manifest
+  This build is running using a custom Elasticsearch snapshot.
+
+  ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST
+
+  To use this locally, simply prefix your commands with:
+
+  \`\`\`
+  ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST"
+  \`\`\`
+
+  e.g.
+
+  \`\`\`
+  ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" node scripts/functional_tests_server.js
+  \`\`\`
+EOF
+  fi
+}
+
+# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build
+{
+  if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+    ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')}
+    export ES_SNAPSHOT_MANIFEST
+    echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST"
+  fi
+}
+
+# Setup CI Stats
+{
+  CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"
+  export CI_STATS_BUILD_ID
+
+  CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)"
+  export CI_STATS_TOKEN
+
+  CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)"
+  export CI_STATS_HOST
+
+  if [[ "$CI_STATS_BUILD_ID" ]]; then
+    echo "CI Stats Build ID: $CI_STATS_BUILD_ID"
+
+    KIBANA_CI_STATS_CONFIG=$(jq -n \
+      --arg buildId "$CI_STATS_BUILD_ID" \
+      --arg apiUrl "https://$CI_STATS_HOST" \
+      --arg apiToken "$CI_STATS_TOKEN" \
+      '{buildId: $buildId, apiUrl: $apiUrl, apiToken: $apiToken}' \
+    )
+    export KIBANA_CI_STATS_CONFIG
+  fi
+}
+
+# Set up misc keys
+{
+  KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value)
+  export KIBANA_CI_REPORTER_KEY
+
+  EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)"
+  export EC_API_KEY
+
+  PROJECT_API_KEY="$(vault_get kibana-ci-project-deploy pr_deploy_api_key)"
+  export PROJECT_API_KEY
+
+  PROJECT_API_DOMAIN="$(vault_get kibana-ci-project-deploy pr_deploy_domain)"
+  export PROJECT_API_DOMAIN
+
+  SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)"
+  export SYNTHETICS_SERVICE_USERNAME
+
+  SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)"
+  export SYNTHETICS_SERVICE_PASSWORD
+
+  SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)"
+  export SYNTHETICS_SERVICE_MANIFEST
+
+  SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)"
+  export SYNTHETICS_REMOTE_KIBANA_USERNAME
+
+  SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)"
+  export SYNTHETICS_REMOTE_KIBANA_PASSWORD
+
+  SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"}
+  export SYNTHETICS_REMOTE_KIBANA_URL
+
+  DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"}
+  export DEPLOY_TAGGER_SLACK_WEBHOOK_URL
+
+  SONAR_LOGIN=$(vault_get sonarqube token)
+  export SONAR_LOGIN
+}
+
+# Set up GCS Service Account for CDN
+{
+  GCS_SA_CDN_KEY="$(vault_get gcs-sa-cdn-prod key)"
+  export GCS_SA_CDN_KEY
+
+  GCS_SA_CDN_EMAIL="$(vault_get gcs-sa-cdn-prod email)"
+  export GCS_SA_CDN_EMAIL
+
+  GCS_SA_CDN_BUCKET="$(vault_get gcs-sa-cdn-prod bucket)"
+  export GCS_SA_CDN_BUCKET
+
+  GCS_SA_CDN_URL="$(vault_get gcs-sa-cdn-prod cdn)"
+  export GCS_SA_CDN_URL
+}
+
+# Setup Failed Test Reporter Elasticsearch credentials
+{
+  TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id)
+  export TEST_FAILURES_ES_CLOUD_ID
+
+  TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username)
+  export TEST_FAILURES_ES_USERNAME
+
+  TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password)
+  export TEST_FAILURES_ES_PASSWORD
+}
+
+# Setup Bazel Remote/Local Cache Credentials
+{
+  BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
+  export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
+  vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
+
+  BAZEL_REMOTE_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-gcs.json"
+  export BAZEL_REMOTE_CACHE_CREDENTIALS_FILE
+  vault_get kibana-ci-bazel-remote-cache-sa-key key | base64 -d > "$BAZEL_REMOTE_CACHE_CREDENTIALS_FILE"
+}
+
+# Setup GCS Service Account Proxy for CI
+{
+  KIBANA_SERVICE_ACCOUNT_PROXY_KEY="$(mktemp -d)/kibana-gcloud-service-account.json"
+  export KIBANA_SERVICE_ACCOUNT_PROXY_KEY
+  vault_get kibana-ci-sa-proxy-key key | base64 -d > "$KIBANA_SERVICE_ACCOUNT_PROXY_KEY"
+}
+
+PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
+if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then
+  source "$PIPELINE_PRE_COMMAND"
+fi

.buildkite/scripts/common/util.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh"
+
 checks-reporter-with-killswitch() {
   if [ "$CHECKS_REPORTER_ACTIVE" == "true" ] ; then
     yarn run github-checks-reporter "$@"

.buildkite/scripts/common/vault_fns.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then
+  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+  VAULT_KV_PREFIX="secret/kibana-issues/dev"
+  IS_LEGACY_VAULT_ADDR=true
+else
+  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+  VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
+  IS_LEGACY_VAULT_ADDR=false
+fi
+export IS_LEGACY_VAULT_ADDR
+
+retry() {
+  local retries=$1; shift
+  local delay=$1; shift
+  local attempts=1
+
+  until "$@"; do
+    retry_exit_status=$?
+    echo "Exited with $retry_exit_status" >&2
+    if (( retries == "0" )); then
+      return $retry_exit_status
+    elif (( attempts == retries )); then
+      echo "Failed $attempts retries" >&2
+      return $retry_exit_status
+    else
+      echo "Retrying $((retries - attempts)) more times..." >&2
+      attempts=$((attempts + 1))
+      sleep "$delay"
+    fi
+  done
+}
+
+vault_get() {
+  key_path=${1:-}
+  field=${2:-}
+
+  fullPath="$VAULT_PATH_PREFIX/$key_path"
+
+  if [[ -z "$field" || "$field" =~ ^-.* ]]; then
+    retry 5 5 vault read "$fullPath" "${@:2}"
+  else
+    retry 5 5 vault read -field="$field" "$fullPath" "${@:3}"
+  fi
+}
+
+vault_set() {
+  key_path=$1
+  shift
+  fields=("$@")
+
+
+  fullPath="$VAULT_PATH_PREFIX/$key_path"
+
+  # shellcheck disable=SC2068
+  retry 5 5 vault write "$fullPath" ${fields[@]}
+}
+
+vault_kv_set() {
+  kv_path=$1
+  shift
+  fields=("$@")
+
+  vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}"
+}

.buildkite/scripts/lifecycle/post_command.sh

@@ -2,6 +2,10 @@
 
 set -euo pipefail
 
+echo '--- Log out of gcloud'
+./.buildkite/scripts/common/activate_service_account.sh --unset-impersonation || echo "Failed to unset impersonation"
+./.buildkite/scripts/common/activate_service_account.sh --logout-gcloud || echo "Failed to log out of gcloud"
+
 echo '--- Agent Debug Info'
 node .buildkite/scripts/lifecycle/print_agent_links.js || true