feat: add token requests client (#805)

* feat: add token requests client

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

* test: add e2e test with e2e-provider and add helm config

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

* chore: move token request rbac markers to separate pkg

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

* test: use multiple test functions

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

* chore: set resync duration at caller

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

Makefile

@@ -404,6 +404,7 @@ e2e-deploy-manifest:
 	kubectl apply -f manifest_staging/deploy/rbac-secretproviderclass.yaml
 	kubectl apply -f manifest_staging/deploy/rbac-secretproviderrotation.yaml
 	kubectl apply -f manifest_staging/deploy/rbac-secretprovidersyncing.yaml
+	kubectl apply -f manifest_staging/deploy/rbac-secretprovidertokenrequest.yaml
 	kubectl apply -f manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasses.yaml
 	kubectl apply -f manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml
 	kubectl apply -f manifest_staging/deploy/role-secretproviderclasses-admin.yaml
@@ -429,7 +430,9 @@ e2e-helm-deploy:
 		--set linux.enabled=true \
 		--set syncSecret.enabled=true \
 		--set enableSecretRotation=true \
-		--set rotationPollInterval=30s
+		--set rotationPollInterval=30s \
+		--set tokenRequests[0].audience="aud1" \
+		--set tokenRequests[1].audience="aud2"
 
 .PHONY: e2e-helm-upgrade
 e2e-helm-upgrade:
@@ -517,6 +520,14 @@ manifests: $(CONTROLLER_GEN) $(KUSTOMIZE)
 	@sed -i '1s/^/{{ if .Values.enableSecretRotation }}\n/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-rotation.yaml
 	@sed -i '1s/^/{{ if .Values.enableSecretRotation }}\n/gm; s/namespace: .*/namespace: {{ .Release.Namespace }}/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-rotation_binding.yaml
 
+	# Generate token requests specific RBAC
+	$(CONTROLLER_GEN) rbac:roleName=secretprovidertokenrequest-role paths="./controllers/tokenrequest" output:dir=config/rbac-tokenrequest
+	$(KUSTOMIZE) build config/rbac-tokenrequest -o manifest_staging/deploy/rbac-secretprovidertokenrequest.yaml
+	cp config/rbac-tokenrequest/role.yaml manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest.yaml
+	cp config/rbac-tokenrequest/role_binding.yaml manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest_binding.yaml
+	@sed -i '1s/^/{{ if .Values.tokenRequests }}\n/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest.yaml
+	@sed -i '1s/^/{{ if .Values.tokenRequests }}\n/gm; s/namespace: .*/namespace: {{ .Release.Namespace }}/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest_binding.yaml
+
 .PHONY: generate-protobuf
 generate-protobuf: $(PROTOC) $(PROTOC_GEN_GO) # generates protobuf
 	$(PROTOC) -I . provider/v1alpha1/service.proto --go_out=plugins=grpc:. --plugin=$(PROTOC_GEN_GO)

cmd/secrets-store-csi-driver/main.go

@@ -27,6 +27,7 @@ import (
 
 	secretsstorev1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"
 	"sigs.k8s.io/secrets-store-csi-driver/controllers"
+	"sigs.k8s.io/secrets-store-csi-driver/pkg/k8s"
 	"sigs.k8s.io/secrets-store-csi-driver/pkg/metrics"
 	"sigs.k8s.io/secrets-store-csi-driver/pkg/rotation"
 	secretsstore "sigs.k8s.io/secrets-store-csi-driver/pkg/secrets-store"
@@ -38,6 +39,7 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	json "k8s.io/component-base/logs/json"
@@ -82,7 +84,8 @@ func main() {
 	flag.Parse()
 
 	if *logFormatJSON {
-		klog.SetLogger(json.JSONLogger)
+		logger, _ := json.NewJSONLogger(nil, nil)
+		klog.SetLogger(logger)
 	}
 	if *enableProfile {
 		klog.InfoS("Starting profiling", "port", *profilePort)
@@ -177,17 +180,29 @@ func main() {
 		reconciler.RunPatcher(ctx)
 	}()
 
+	// token request client
+	kubeClient := kubernetes.NewForConfigOrDie(cfg)
+	tokenClient := k8s.NewTokenClient(kubeClient, *driverName, 10*time.Minute)
+	if err != nil {
+		klog.ErrorS(err, "failed to create token client")
+		os.Exit(1)
+	}
+	if err = tokenClient.Run(ctx.Done()); err != nil {
+		klog.ErrorS(err, "failed to run token client")
+		os.Exit(1)
+	}
+
 	// Secret rotation
 	if *enableSecretRotation {
-		rec, err := rotation.NewReconciler(mgr.GetCache(), scheme, *providerVolumePath, *nodeID, *rotationPollInterval, providerClients)
+		rec, err := rotation.NewReconciler(mgr.GetCache(), scheme, *providerVolumePath, *nodeID, *rotationPollInterval, providerClients, tokenClient)
 		if err != nil {
 			klog.ErrorS(err, "failed to initialize rotation reconciler")
 			os.Exit(1)
 		}
 		go rec.Run(ctx.Done())
 	}
 
-	driver := secretsstore.NewSecretsStoreDriver(*driverName, *nodeID, *endpoint, *providerVolumePath, providerClients, mgr.GetClient(), mgr.GetAPIReader())
+	driver := secretsstore.NewSecretsStoreDriver(*driverName, *nodeID, *endpoint, *providerVolumePath, providerClients, mgr.GetClient(), mgr.GetAPIReader(), tokenClient)
 	driver.Run(ctx)
 }
 

config/rbac-tokenrequest/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- role.yaml
+- role_binding.yaml

config/rbac-tokenrequest/role.yaml

@@ -0,0 +1,14 @@
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: secretprovidertokenrequest-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create

config/rbac-tokenrequest/role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: secretprovidertokenrequest-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: secretprovidertokenrequest-role
+subjects:
+- kind: ServiceAccount
+  name: secrets-store-csi-driver
+  namespace: kube-system

config/rbac/role.yaml

@@ -49,3 +49,13 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - storage.k8s.io
+  resourceNames:
+  - secrets-store.csi.k8s.io
+  resources:
+  - csidrivers
+  verbs:
+  - get
+  - list
+  - watch

controllers/secretproviderclasspodstatus_controller.go

@@ -213,6 +213,7 @@ func (r *SecretProviderClassPodStatusReconciler) ListOptionsLabelSelector() clie
 // +kubebuilder:rbac:groups=secrets-store.csi.x-k8s.io,resources=secretproviderclasses,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
+// +kubebuilder:rbac:groups="storage.k8s.io",resources=csidrivers,verbs=get;list;watch,resourceNames=secrets-store.csi.k8s.io
 
 func (r *SecretProviderClassPodStatusReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	r.mutex.Lock()

controllers/tokenrequest/tokenrequest.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package tokenrequest holds the RBAC permission annotations for the controller
+// to create a serviceaccount token and pass it as part of Mount Request.
+// ref: https://kubernetes-csi.github.io/docs/token-requests.html
+package tokenrequest
+
+// +kubebuilder:rbac:groups="",resources="serviceaccounts/token",verbs=create

go.mod

@@ -8,46 +8,50 @@ require (
 	github.com/google/go-cmp v0.5.5
 	github.com/kubernetes-csi/csi-lib-utils v0.10.0
 	github.com/kubernetes-csi/csi-test/v4 v4.2.0
-	github.com/onsi/gomega v1.13.0
+	github.com/onsi/gomega v1.17.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/stretchr/testify v1.7.0
 	go.opentelemetry.io/otel v0.20.0
 	go.opentelemetry.io/otel/exporters/metric/prometheus v0.20.0
 	go.opentelemetry.io/otel/metric v0.20.0
-	google.golang.org/grpc v1.39.0
-	google.golang.org/protobuf v1.26.0
-	k8s.io/api v0.22.3
-	k8s.io/apimachinery v0.22.3
-	k8s.io/client-go v0.22.3
-	k8s.io/component-base v0.22.3
-	k8s.io/klog/v2 v2.10.0
-	k8s.io/mount-utils v0.22.3
-	sigs.k8s.io/controller-runtime v0.9.0
+	google.golang.org/grpc v1.40.0
+	google.golang.org/protobuf v1.27.1
+	k8s.io/api v0.23.0
+	k8s.io/apimachinery v0.23.0
+	k8s.io/client-go v0.23.0
+	k8s.io/component-base v0.23.0
+	k8s.io/klog/v2 v2.30.0
+	k8s.io/kubernetes v1.23.0
+	k8s.io/mount-utils v0.23.0
+	sigs.k8s.io/controller-runtime v0.11.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-logr/logr v1.2.0 // indirect
+	github.com/go-logr/zapr v1.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.26.0 // indirect
+	github.com/prometheus/common v0.28.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opentelemetry.io/otel/sdk v0.20.0 // indirect
@@ -56,24 +60,56 @@ require (
 	go.opentelemetry.io/otel/trace v0.20.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
-	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
-	golang.org/x/text v0.3.6 // indirect
+	go.uber.org/zap v1.19.1 // indirect
+	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
+	google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/apiextensions-apiserver v0.21.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect
-	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
+	k8s.io/apiextensions-apiserver v0.23.0 // indirect
+	k8s.io/apiserver v0.23.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+replace (
+	k8s.io/api => k8s.io/api v0.23.0
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.23.0
+	k8s.io/apiserver => k8s.io/apiserver v0.23.0
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.0
+	k8s.io/client-go => k8s.io/client-go v0.23.0
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.0
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.0
+	k8s.io/code-generator => k8s.io/code-generator v0.23.0
+	k8s.io/component-base => k8s.io/component-base v0.23.0
+	k8s.io/component-helpers => k8s.io/component-helpers v0.23.0
+	k8s.io/controller-manager => k8s.io/controller-manager v0.23.0
+	k8s.io/cri-api => k8s.io/cri-api v0.23.0
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.0
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.0
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.0
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.0
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.0
+	k8s.io/kubectl => k8s.io/kubectl v0.23.0
+	k8s.io/kubelet => k8s.io/kubelet v0.23.0
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.0
+	k8s.io/metrics => k8s.io/metrics v0.23.0
+	k8s.io/mount-utils => k8s.io/mount-utils v0.23.0
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.0
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.0
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.23.0
+	k8s.io/sample-controller => k8s.io/sample-controller v0.23.0
 )