Use managed identity for aks pipeline

* Update azure pipeline config * Update e2e to support managed identity
k8s-infra-cherrypick-robot · May 8, 2024 · d29f718 · d29f718
1 parent 85a5e5f
commit d29f718
Show file tree

Hide file tree

Showing 12 changed files with 79 additions and 32 deletions.
diff --git a/.pipelines/ccm-image.yml b/.pipelines/ccm-image.yml
@@ -6,7 +6,11 @@ steps:
       git show --stat
       echo $REGISTRY_PASSWORD | docker login $REGISTRY_URL -u $REGISTRY_USERNAME --password-stdin
       export IMAGE_REGISTRY=$REGISTRY_URL
-      export GOPATH="/home/vsts/go"
+      USER="cloudtest"
+      if [[ -n "${RELEASE_PIPELINE:-}" ]]; then
+        USER="vsts"
+      fi
+      export GOPATH="/home/${USER}/go"
       export PATH="${PATH}:${GOPATH}/bin"
 
       if [[ ! -d kubetest2-aks ]]; then
@@ -19,7 +23,7 @@ steps:
       go install sigs.k8s.io/kubetest2@latest
       go mod tidy
       make deployer
-      sudo GOPATH="/home/vsts/go" make install
+      sudo GOPATH="/home/${USER}/go" make install
       popd
 
       export IMAGE_TAG="$(git describe --tags --match "v[0-9].*")"

diff --git a/.pipelines/cnm-image.yml b/.pipelines/cnm-image.yml
@@ -6,7 +6,11 @@ steps:
       git show --stat
       echo $REGISTRY_PASSWORD | docker login $REGISTRY_URL -u $REGISTRY_USERNAME --password-stdin
       export IMAGE_REGISTRY=$REGISTRY_URL
-      export GOPATH="/home/vsts/go"
+      USER="cloudtest"
+      if [[ -n "${RELEASE_PIPELINE:-}" ]]; then
+        USER="vsts"
+      fi
+      export GOPATH="/home/${USER}/go"
       export PATH="${PATH}:${GOPATH}/bin"
 
       if [[ ! -d kubetest2-aks ]]; then
@@ -19,7 +23,7 @@ steps:
       go install sigs.k8s.io/kubetest2@latest
       go mod tidy
       make deployer
-      sudo GOPATH="/home/vsts/go" make install
+      sudo GOPATH="/home/${USER}/go" make install
       popd
 
       export IMAGE_TAG="$(git describe --tags --match "v[0-9].*")"

diff --git a/.pipelines/daily-gc.yml b/.pipelines/daily-gc.yml
@@ -10,6 +10,8 @@ trigger: none
 
 pr: none
 
+pool: cloud-provider-azure-e2e-pool
+
 variables:
 - template: var-e2e.yml
 
@@ -23,13 +25,11 @@ stages:
         - bash: |
             export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID
             export AZURE_TENANT_ID=$AZ_TENANT_ID
-            export AZURE_CLIENT_ID=$SP_CLIENT_ID
-            export AZURE_CLIENT_SECRET=$SP_CLIENT_SECRET
+            export AZURE_MANAGED_IDENTITY_CLIENT_ID=$AZ_MANAGED_IDENTITY_CLIENT_ID
 
             .pipelines/scripts/daily-gc.sh
           displayName: gc aks and resource groups
           env:
             AZ_SUBSCRIPTION_ID: $(az.subscription_id)
             AZ_TENANT_ID: $(az.tenant_id)
-            SP_CLIENT_ID: $(sp.client_id)
-            SP_CLIENT_SECRET: $(sp.client_secret)
+            AZ_MANAGED_IDENTITY_PRINCIPAL_ID: $(az.mi_client_id)
diff --git a/.pipelines/run-autoscaling-e2e.yml b/.pipelines/run-autoscaling-e2e.yml
@@ -9,8 +9,7 @@ trigger: none
 
 pr: none
 
-pool:
-  vmImage: ubuntu-latest
+pool: cloud-provider-azure-e2e-pool
 
 variables:
 - template: var-e2e.yml

diff --git a/.pipelines/run-autoscaling-multipool-e2e.yml b/.pipelines/run-autoscaling-multipool-e2e.yml
@@ -9,8 +9,7 @@ trigger: none
 
 pr: none
 
-pool:
-  vmImage: ubuntu-latest
+pool: cloud-provider-azure-e2e-pool
 
 variables:
 - template: var-e2e.yml

diff --git a/.pipelines/run-basic-lb-e2e.yml b/.pipelines/run-basic-lb-e2e.yml
@@ -12,8 +12,7 @@ trigger:
 
 pr: none
 
-pool:
-  vmImage: ubuntu-latest
+pool: cloud-provider-azure-e2e-pool
 
 variables:
 - template: var-e2e.yml

diff --git a/.pipelines/run-e2e.yml b/.pipelines/run-e2e.yml
@@ -8,9 +8,8 @@ steps:
   - bash: |
       export IMAGE_REGISTRY=$REGISTRY_URL
       export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID
+      export AZURE_MANAGED_IDENTITY_CLIENT_ID=$AZ_MANAGED_IDENTITY_CLIENT_ID
       export AZURE_TENANT_ID=$AZ_TENANT_ID
-      export AZURE_CLIENT_ID=$SP_CLIENT_ID
-      export AZURE_CLIENT_SECRET=$SP_CLIENT_SECRET
       export ARTIFACT_DIR=$BUILD_ARTIFACT_STAGING_DIRECTORY
       export KUSTO_INGESTION_URI=$KUSTO_INGESTION_URI
 
@@ -24,9 +23,8 @@ steps:
     env:
       REGISTRY_URL: $(registry.url)
       AZ_SUBSCRIPTION_ID: $(az.subscription_id)
+      AZ_MANAGED_IDENTITY_PRINCIPAL_ID: $(az.mi_client_id)
       AZ_TENANT_ID: $(az.tenant_id)
-      SP_CLIENT_ID: $(sp.client_id)
-      SP_CLIENT_SECRET: $(sp.client_secret)
       BUILD_ARTIFACT_STAGING_DIRECTORY: $(Build.ArtifactStagingDirectory)
       BUILD_SOURCE_BRANCH_NAME: $(Build.SourceBranchName)
       KUSTO_INGESTION_URI: $(kusto.ingestion_uri)

diff --git a/.pipelines/run-vmas-e2e.yml b/.pipelines/run-vmas-e2e.yml
@@ -9,8 +9,7 @@ trigger: none
 
 pr: none
 
-pool:
-  vmImage: ubuntu-latest
+pool: cloud-provider-azure-e2e-pool
 
 variables:
 - template: var-e2e.yml

diff --git a/.pipelines/scripts/daily-gc.sh b/.pipelines/scripts/daily-gc.sh
@@ -21,7 +21,7 @@ set -o pipefail
 CLUSTER_NAME="aks-cluster"
 CURRENT_DATE="$(date +%s)"
 
-az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}"
+az login --identity --username "${AZURE_MANAGED_IDENTITY_CLIENT_ID:-}"
 az group list --tag usage=aks-cluster-e2e | jq -r '.[].name' | awk '{print $1}' | while read -r RESOURCE_GROUP; do
   RG_DATE="$(az group show --resource-group ${RESOURCE_GROUP} | jq -r '.tags.creation_date')"
   DATE_DIFF="$(expr ${CURRENT_DATE} - ${RG_DATE})"

diff --git a/.pipelines/scripts/run-e2e.sh b/.pipelines/scripts/run-e2e.sh
@@ -19,14 +19,20 @@ set -o nounset
 set -o pipefail
 
 REPO_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/../..)
-export GOPATH="/home/vsts/go"
-export PATH="${PATH:-}:${GOPATH}/bin"
-export AKS_CLUSTER_ID="/subscriptions/${AZURE_SUBSCRIPTION_ID:-}/resourcegroups/${RESOURCE_GROUP:-}/providers/Microsoft.ContainerService/managedClusters/${CLUSTER_NAME:-}"
-
-if [[ -z "${RELEASE_PIPELINE:-}" ]]; then
-  az login --service-principal -u "${AZURE_CLIENT_ID:-}" -p "${AZURE_CLIENT_SECRET:-}" --tenant "${AZURE_TENANT_ID:-}"
+USER="cloudtest"
+if [[ -n "${RELEASE_PIPELINE:-}" ]]; then
+  # release pipeline uses sp
+  USER="vsts"
+else
+  # aks pipeline uses managed identity
+  az login --identity --username "${AZURE_MANAGED_IDENTITY_CLIENT_ID:-}"
+  export E2E_MANAGED_IDENTITY_TYPE="userassigned"
 fi
 
+export GOPATH="/home/${USER}/go"
+export PATH="${PATH}:${GOPATH}/bin"
+export AKS_CLUSTER_ID="/subscriptions/${AZURE_SUBSCRIPTION_ID:-}/resourcegroups/${RESOURCE_GROUP:-}/providers/Microsoft.ContainerService/managedClusters/${CLUSTER_NAME:-}"
+
 get_random_location() {
   local LOCATIONS=("eastus")
   echo "${LOCATIONS[${RANDOM} % ${#LOCATIONS[@]}]}"
@@ -125,7 +131,7 @@ if [[ "${SKIP_BUILD_KUBETEST2_AKS:-}" != "true" ]]; then
   if [[ -n "${RELEASE_PIPELINE:-}" ]]; then
     make install
   else
-    sudo GOPATH="/home/vsts/go" make install
+    sudo GOPATH="/home/${USER}/go" make install
   fi
   rm /tmp/cloud-provider-azure -rf
   popd
@@ -181,6 +187,13 @@ if [[ "${CLUSTER_TYPE:-}" =~ "autoscaling" ]]; then
   fi
 fi
 
+# Do role assignment for the managed identity
+if [[ -z "${RELEASE_PIPELINE:-}" ]]; then
+  MC_RESOURCE_ID="/subscriptions/${AZURE_SUBSCRIPTION_ID:-}/resourceGroups/MC_${RESOURCE_GROUP:-}_${CLUSTER_NAME:-}_eastus"
+  AGENTPOOL_MI_PRINCIPAL_ID="$(az identity show -n ${CLUSTER_NAME:-}-agentpool -g MC_${RESOURCE_GROUP:-}_${CLUSTER_NAME:-}_eastus --subscription ${AZURE_SUBSCRIPTION_ID:-} | jq -r '.principalId')"
+  az role assignment create --assignee "${AGENTPOOL_MI_PRINCIPAL_ID}" --role "AcrPull" --scope "${MC_RESOURCE_ID}" --subscription "${AZURE_SUBSCRIPTION_ID:-}"
+fi
+
 if [[ "${SKIP_E2E:-}" != "true" ]]; then
   echo "Running e2e"
   export E2E_ON_AKS_CLUSTER=true

diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go
@@ -100,6 +100,13 @@ func TestAzureTest(t *testing.T) {
 	if !strings.EqualFold(os.Getenv(utils.IngestTestResult), utils.TrueValue) {
 		return
 	}
+
+	// TODO: Use managed identity for Kusto ingestion
+	skipKustoIngestion := true
+	if skipKustoIngestion {
+		return
+	}
+
 	klog.Infof("Ingesting test result to kusto")
 	if err := utils.KustoIngest(passed, suiteConfig.LabelFilter, os.Getenv(utils.AKSClusterType), reporterConfig.JUnitReport); err != nil {
 		klog.Error(err)

diff --git a/tests/e2e/utils/azure_auth.go b/tests/e2e/utils/azure_auth.go
@@ -17,6 +17,7 @@ limitations under the License.
 package utils
 
 import (
+	"fmt"
 	"os"
 
 	"sigs.k8s.io/cloud-provider-azure/pkg/azclient"
@@ -31,6 +32,10 @@ const (
 	ClusterLocationEnv        = "AZURE_LOCATION"
 	ClusterEnvironment        = "AZURE_ENVIRONMENT"
 	LoadBalancerSkuEnv        = "AZURE_LOADBALANCER_SKU"
+	managedIdentityClientID   = "AZURE_MANAGED_IDENTITY_CLIENT_ID"
+	managedIdentityType       = "E2E_MANAGED_IDENTITY_TYPE"
+
+	userAssignedManagedIdentity = "userassigned"
 )
 
 // azureAuthConfigFromTestProfile obtains azure config from Environment
@@ -39,10 +44,30 @@ func azureAuthConfigFromTestProfile() (*azclient.AzureAuthConfig, *azclient.ARMC
 	if len(envStr) == 0 {
 		envStr = "AZUREPUBLICCLOUD"
 	}
-	return &azclient.AzureAuthConfig{
-			AADClientID:     os.Getenv(ServicePrincipleIDEnv),
-			AADClientSecret: os.Getenv(ServicePrincipleSecretEnv),
-		}, &azclient.ARMClientConfig{
+
+	var azureAuthConfig azclient.AzureAuthConfig
+	servicePrincipleIDEnv := os.Getenv(ServicePrincipleIDEnv)
+	servicePrincipleSecretEnv := os.Getenv(ServicePrincipleSecretEnv)
+	managedIdentityTypeEnv := os.Getenv(managedIdentityType)
+	managedIdentityClientIDEnv := os.Getenv(managedIdentityClientID)
+	if servicePrincipleIDEnv != "" && servicePrincipleSecretEnv != "" {
+		azureAuthConfig = azclient.AzureAuthConfig{
+			AADClientID:     servicePrincipleIDEnv,
+			AADClientSecret: servicePrincipleSecretEnv,
+		}
+	} else if managedIdentityTypeEnv != "" {
+		azureAuthConfig = azclient.AzureAuthConfig{
+			UseManagedIdentityExtension: true,
+		}
+		if managedIdentityTypeEnv == userAssignedManagedIdentity {
+			azureAuthConfig.UserAssignedIdentityID = managedIdentityClientIDEnv
+
+		}
+	} else {
+		return nil, nil, nil, fmt.Errorf("failed to get Azure auth config from environment")
+	}
+
+	return &azureAuthConfig, &azclient.ARMClientConfig{
 			Cloud:    envStr,
 			TenantID: os.Getenv(TenantIDEnv),
 		}, &azclient.ClientFactoryConfig{