🛡 Invalidate ServiceAccounts for controllers part of `kube-controll…

…er-manager` (gardener#5422)

* Add component boilerplate for shoot system resources

* Add hack script for fetching KCM controller names

Similar to `hack/compare-k8s-feature-gates.sh`

* Invalidate KCM controller `ServiceAccount`s in shoot

* Add unit tests

hack/compare-kcm-controllers.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+  echo "Usage:"
+  echo "> compare-kcm-controllers.sh [ -h | <old version> <new version> ]"
+  echo
+  echo ">> For example: compare-kcm-controllers.sh 1.22 1.23"
+
+  exit 0
+}
+
+if [ "$1" == "-h" ] || [ "$#" -ne 2 ]; then
+  usage
+fi
+
+versions=("$1" "$2")
+
+out_dir=dev/temp
+mkdir -p "${out_dir}"
+
+for version in "${versions[@]}"; do
+  rm -rf "${out_dir}/kubernetes-${version}"
+  rm -f "${out_dir}/kcm-controllers-${version}.txt"
+
+  git clone --depth 1 --filter=blob:none --sparse https://github.com/kubernetes/kubernetes -b "release-${version}" "${out_dir}/kubernetes-${version}"
+  pushd "${out_dir}/kubernetes-${version}" > /dev/null
+  git sparse-checkout set "cmd/kube-controller-manager/app"
+  popd > /dev/null
+
+  cat "${out_dir}/kubernetes-${version}/cmd/kube-controller-manager/app/"*.go |\
+    sed -rn "s/.*[Client|Config]OrDie\(\"(.*)\"\).*/\1/p" |\
+    grep -vE "informers|discovery" |\
+    sort |\
+    uniq > "${out_dir}/kcm-controllers-${version}.txt"
+done
+
+echo
+echo "kube-controller-manager controllers added in $2 compared to $1:"
+diff "${out_dir}/kcm-controllers-$1.txt" "${out_dir}/kcm-controllers-$2.txt" | grep '>' | awk '{print $2}'
+echo
+echo "kube-controller-manager controllers removed in $2 compared to $1:"
+diff "${out_dir}/kcm-controllers-$1.txt" "${out_dir}/kcm-controllers-$2.txt" | grep '<' | awk '{print $2}'

pkg/gardenlet/controller/shoot/shoot_control_reconcile.go

@@ -410,6 +410,11 @@ func (r *shootReconciler) runReconcileShootFlow(ctx context.Context, o *operatio
 			Fn:           flow.TaskFn(botanist.Shoot.Components.SystemComponents.Namespaces.Deploy).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.HibernationEnabled),
 			Dependencies: flow.NewTaskIDs(deployGardenerResourceManager, waitUntilOperatingSystemConfigReady),
 		})
+		_ = g.Add(flow.Task{
+			Name:         "Deploying shoot system resources",
+			Fn:           flow.TaskFn(botanist.Shoot.Components.SystemComponents.Resources.Deploy).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.HibernationEnabled),
+			Dependencies: flow.NewTaskIDs(deployGardenerResourceManager, waitUntilOperatingSystemConfigReady),
+		})
 		_ = g.Add(flow.Task{
 			Name: "Deploying CoreDNS system component",
 			Fn: flow.TaskFn(func(ctx context.Context) error {

pkg/operation/botanist/botanist.go

@@ -159,6 +159,7 @@ func New(ctx context.Context, o *operation.Operation) (*Botanist, error) {
 	if err != nil {
 		return nil, err
 	}
+	o.Shoot.Components.SystemComponents.Resources = b.DefaultShootSystem()
 	o.Shoot.Components.SystemComponents.VPNShoot, err = b.DefaultVPNShoot()
 	if err != nil {
 		return nil, err

pkg/operation/botanist/component/shootsystem/shootsystem.go

@@ -0,0 +1,177 @@
+// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shootsystem
+
+import (
+	"context"
+	"time"
+
+	resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1"
+	"github.com/gardener/gardener/pkg/client/kubernetes"
+	"github.com/gardener/gardener/pkg/operation/botanist/component"
+	"github.com/gardener/gardener/pkg/utils/managedresources"
+	versionutils "github.com/gardener/gardener/pkg/utils/version"
+
+	"github.com/Masterminds/semver"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// ManagedResourceName is the name of the ManagedResource containing the resource specifications.
+	ManagedResourceName = "shoot-core-system"
+)
+
+// Values is a set of configuration values for the system resources.
+type Values struct {
+	// KubernetesVersion is the Kubernetes version of the shoot cluster.
+	KubernetesVersion *semver.Version
+}
+
+// New creates a new instance of DeployWaiter for shoot system resources.
+func New(
+	client client.Client,
+	namespace string,
+	values Values,
+) component.DeployWaiter {
+	return &shootSystem{
+		client:    client,
+		namespace: namespace,
+		values:    values,
+	}
+}
+
+type shootSystem struct {
+	client    client.Client
+	namespace string
+	values    Values
+}
+
+func (s *shootSystem) Deploy(ctx context.Context) error {
+	data, err := s.computeResourcesData()
+	if err != nil {
+		return err
+	}
+
+	return managedresources.CreateForShoot(ctx, s.client, s.namespace, ManagedResourceName, false, data)
+}
+
+func (s *shootSystem) Destroy(ctx context.Context) error {
+	return managedresources.DeleteForShoot(ctx, s.client, s.namespace, ManagedResourceName)
+}
+
+// TimeoutWaitForManagedResource is the timeout used while waiting for the ManagedResources to become healthy
+// or deleted.
+var TimeoutWaitForManagedResource = 2 * time.Minute
+
+func (s *shootSystem) Wait(ctx context.Context) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, TimeoutWaitForManagedResource)
+	defer cancel()
+
+	return managedresources.WaitUntilHealthy(timeoutCtx, s.client, s.namespace, ManagedResourceName)
+}
+
+func (s *shootSystem) WaitCleanup(ctx context.Context) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, TimeoutWaitForManagedResource)
+	defer cancel()
+
+	return managedresources.WaitUntilDeleted(timeoutCtx, s.client, s.namespace, ManagedResourceName)
+}
+
+func (s *shootSystem) computeResourcesData() (map[string][]byte, error) {
+	var (
+		registry = managedresources.NewRegistry(kubernetes.ShootScheme, kubernetes.ShootCodec, kubernetes.ShootSerializer)
+
+		kubeControllerManagerServiceAccounts []client.Object
+	)
+
+	for _, name := range s.getServiceAccountNamesToInvalidate() {
+		kubeControllerManagerServiceAccounts = append(kubeControllerManagerServiceAccounts, &corev1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        name,
+				Namespace:   metav1.NamespaceSystem,
+				Annotations: map[string]string{resourcesv1alpha1.KeepObject: "true"},
+			},
+			AutomountServiceAccountToken: pointer.Bool(false),
+		})
+	}
+
+	return registry.AddAllAndSerialize(kubeControllerManagerServiceAccounts...)
+}
+
+func (s *shootSystem) getServiceAccountNamesToInvalidate() []string {
+	// Well-known {kube,cloud}-controller-manager controllers using a token for ServiceAccounts in the shoot
+	// To maintain this list for each new Kubernetes version:
+	// * Run hack/compare-kcm-controllers.sh <old-version> <new-version> (e.g. 'hack/compare-kcm-controllers.sh 1.22 1.23').
+	//   It will present 2 lists of controllers: those added and those removed in <new-version> compared to <old-version>.
+	// * Double check whether such ServiceAccount indeed appears in the kube-system namespace when creating a cluster
+	//   with <new-version>. Note that it sometimes might be hidden behind a default-off feature gate.
+	//   If it appears, add all added controllers to the list if the Kubernetes version is high enough.
+	// * For any removed controllers, add them only to the Kubernetes version if it is low enough.
+	kubeControllerManagerServiceAccountNames := []string{
+		"attachdetach-controller",
+		"bootstrap-signer",
+		"certificate-controller",
+		"clusterrole-aggregation-controller",
+		"controller-discovery",
+		"cronjob-controller",
+		"daemon-set-controller",
+		"deployment-controller",
+		"disruption-controller",
+		"endpoint-controller",
+		"endpointslice-controller",
+		"expand-controller",
+		"generic-garbage-collector",
+		"horizontal-pod-autoscaler",
+		"job-controller",
+		"metadata-informers",
+		"namespace-controller",
+		"node-controller",
+		"persistent-volume-binder",
+		"pod-garbage-collector",
+		"pv-protection-controller",
+		"pvc-protection-controller",
+		"replicaset-controller",
+		"replication-controller",
+		"resourcequota-controller",
+		"root-ca-cert-publisher",
+		"route-controller",
+		"service-account-controller",
+		"service-controller",
+		"shared-informers",
+		"statefulset-controller",
+		"token-cleaner",
+		"tokens-controller",
+		"ttl-after-finished-controller",
+		"ttl-controller",
+	}
+
+	if versionutils.ConstraintK8sGreaterEqual119.Check(s.values.KubernetesVersion) {
+		kubeControllerManagerServiceAccountNames = append(kubeControllerManagerServiceAccountNames,
+			"endpointslicemirroring-controller",
+			"ephemeral-volume-controller",
+		)
+	}
+
+	if versionutils.ConstraintK8sGreaterEqual120.Check(s.values.KubernetesVersion) {
+		kubeControllerManagerServiceAccountNames = append(kubeControllerManagerServiceAccountNames,
+			"storage-version-garbage-collector",
+		)
+	}
+
+	return append(kubeControllerManagerServiceAccountNames, "default")
+}

pkg/operation/botanist/component/shootsystem/shootsystem_suite_test.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shootsystem_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestShootSystem(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Botanist Component ShootSystem Suite")
+}