Merge pull request rook#9534 from yati1998/csi-addon

csi: implement csi-addons sidecar

Documentation/ceph-upgrade.md

@@ -552,6 +552,7 @@ ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0"
 ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
 ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
 CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
+ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 ```
 
 ### **Use default images**
@@ -577,4 +578,5 @@ k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
 k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
 quay.io/cephcsi/cephcsi:v3.5.0
 quay.io/csiaddons/volumereplication-operator:v0.1.0
+quay.io/csiaddons/k8s-sidecar:v0.1.0
 ```

Documentation/helm-operator.md

@@ -130,6 +130,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.cephfsGrpcMetricsPort`         | CSI CephFS driver GRPC metrics port.                                                                                        | `9091`                                                    |
 | `csi.cephfsLivenessMetricsPort`     | CSI CephFS driver metrics port.                                                                                             | `9081`                                                    |
 | `csi.rbdGrpcMetricsPort`            | Ceph CSI RBD driver GRPC metrics port.                                                                                      | `9090`                                                    |
+| `csi.csiAddonsPort`            | CSI Addons server port.                                                                                      | `9070`                                                    |
 | `csi.rbdLivenessMetricsPort`        | Ceph CSI RBD driver metrics port.                                                                                           | `8080`                                                    |
 | `csi.forceCephFSKernelClient`       | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs.                                                | `true`                                                    |
 | `csi.kubeletDirPath`                | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag)                                | `/var/lib/kubelet`                                        |
@@ -145,6 +146,8 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.rbdPodLabels`                  | Labels to add to the CSI RBD Pods.                                                                                          | <none>                                                    |
 | `csi.volumeReplication.enabled`     | Enable Volume Replication.                                                                                                  | `false`                                                   |
 | `csi.volumeReplication.image`       | Volume Replication Controller image.                                                                                        | `quay.io/csiaddons/volumereplication-operator:v0.1.0`     |
+| `csi.csiAddons.enabled`     | Enable CSIAddons                                                                                                  | `false`                                                   |
+| `csi.csiAddons.image`       | CSIAddons Sidecar image.                                                                                        | `quay.io/csiaddons/k8s-sidecar:v0.1.0`     |
 | `admissionController.tolerations`   | Array of tolerations in YAML format which will be added to admission controller deployment.                                 | <none>                                                    |
 | `admissionController.nodeAffinity`  | The node labels for affinity of the admission controller deployment (***)                                                   | <none>                                                    |
 | `allowMultipleFilesystems`          | **(experimental in  Octopus (v15))** Allows multiple filesystems to be deployed to a Ceph cluster.                          | `false`                                                   |

build/rbac/rbac.yaml

@@ -757,6 +757,9 @@ spec:
     # Ceph mgr Prometheus Metrics
     - min: 9283
       max: 9283
+    # port for CSIAddons
+    - min: 9061
+      max: 9079
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1

deploy/charts/rook-ceph/templates/deployment.yaml

@@ -91,6 +91,8 @@ spec:
           value: {{ .Values.csi.enableOMAPGenerator | quote }}
         - name: CSI_ENABLE_VOLUME_REPLICATION
           value: {{ .Values.csi.volumeReplication.enabled | quote }}
+        - name: CSI_ENABLE_CSIADDONS
+          value: {{ .Values.csi.csiAddons.enabled | quote }}
         - name: CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT
           value: {{ .Values.csi.enablePluginSelinuxHostMount | quote }}
 {{- if .Values.csi.enableCSIHostNetwork }}
@@ -161,6 +163,16 @@ spec:
           value: {{ .Values.csi.volumeReplication.image | quote }}
 {{- end }}
 {{- end }}
+{{- if .Values.csi.csiAddons }}
+{{- if .Values.csi.csiAddons.image }}
+        - name: ROOK_CSIADDONS_IMAGE
+          value: {{ .Values.csi.csiAddons.image | quote }}
+{{- end }}
+{{- if .Values.csi.csiAddonsPort }}
+        - name: CSIADDONS_PORT
+          value: {{ .Values.csi.csiAddonsPort | quote }}
+{{- end }}
+{{- end }}
 {{- if .Values.csi.cephfsPodLabels }}
         - name: ROOK_CSI_CEPHFS_POD_LABELS
           value: {{ .Values.csi.cephfsPodLabels | quote }}

deploy/charts/rook-ceph/templates/psp.yaml

@@ -76,6 +76,9 @@ spec:
     # Ceph mgr Prometheus Metrics
     - min: 9283
       max: 9283
+    # port for CSIAddons
+    - min: 9061
+      max: 9079
 {{- if .Values.rbacEnable }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1

deploy/charts/rook-ceph/templates/role.yaml

@@ -61,6 +61,18 @@ rules:
     resources: ["leases"]
     verbs: ["get", "watch", "list", "delete", "update", "create"]
 ---
+{{- if .Values.csi.csiAddons }}
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-csi-nodeplugin
+  namespace: {{ .Release.Namespace }} # namespace:operator
+rules:
+  - apiGroups: ["csiaddons.openshift.io"]
+    resources: ["csiaddonsnodes"]
+    verbs: ["create"]
+---
+{{- end }}
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -76,4 +88,9 @@ rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "watch", "list", "delete", "update", "create"]
+  {{- if .Values.csi.csiAddons }}
+  - apiGroups: ["csiaddons.openshift.io"]
+    resources: ["csiaddonsnodes"]
+    verbs: ["create"]
+  {{- end }}
 {{- end }}

deploy/charts/rook-ceph/templates/rolebinding.yaml

@@ -32,6 +32,22 @@ roleRef:
   name: cephfs-external-provisioner-cfg
   apiGroup: rbac.authorization.k8s.io
 ---
+{{- if .Values.csi.csiAddons }}
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-csi-nodeplugin-role-cfg
+  namespace: {{ .Release.Namespace }} # namespace:operator
+subjects:
+  - kind: ServiceAccount
+    name: rook-csi-rbd-plugin-sa
+    namespace: {{ .Release.Namespace }} # namespace:operator
+roleRef:
+  kind: Role
+  name: rbd-csi-nodeplugin
+  apiGroup: rbac.authorization.k8s.io
+---
+{{- end -}}
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:

deploy/charts/rook-ceph/values.yaml

@@ -268,6 +268,7 @@ csi:
   #cephfsGrpcMetricsPort: 9091
   #cephfsLivenessMetricsPort: 9081
   #rbdGrpcMetricsPort: 9090
+  #csiAddonsPort: 9070
   # Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS
   # you may want to disable this setting. However, this will cause an issue during upgrades
   # with the FUSE client. See the upgrade guide: https://rook.io/docs/rook/v1.2/ceph-upgrade.html
@@ -296,7 +297,10 @@ csi:
   volumeReplication:
     enabled: false
     #image: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
-
+  # Enable the CSIAddons sidecar.
+  csiAddons:
+    enabled: false
+    #image: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 enableDiscoveryDaemon: false
 cephCommandsTimeoutSeconds: "15"
 

deploy/examples/common.yaml

@@ -841,6 +841,9 @@ spec:
     # Ceph mgr Prometheus Metrics
     - min: 9283
       max: 9283
+    # port for CSIAddons
+    - min: 9061
+      max: 9079
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
@@ -860,6 +863,16 @@ rules:
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-csi-nodeplugin
+  namespace: rook-ceph # namespace:operator
+rules:
+  - apiGroups: ["csiaddons.openshift.io"]
+    resources: ["csiaddonsnodes"]
+    verbs: ["create"]
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: rbd-external-provisioner-cfg
   namespace: rook-ceph # namespace:operator
@@ -873,6 +886,9 @@ rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "watch", "list", "delete", "update", "create"]
+  - apiGroups: ["csiaddons.openshift.io"]
+    resources: ["csiaddonsnodes"]
+    verbs: ["create"]
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
@@ -1045,6 +1061,20 @@ roleRef:
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: rbd-csi-nodeplugin-role-cfg
+  namespace: rook-ceph # namespace:operator
+subjects:
+  - kind: ServiceAccount
+    name: rook-csi-rbd-plugin-sa
+    namespace: rook-ceph # namespace:operator
+roleRef:
+  kind: Role
+  name: rbd-csi-nodeplugin
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: rbd-csi-provisioner-role-cfg
   namespace: rook-ceph # namespace:operator

deploy/examples/images.txt

@@ -5,5 +5,6 @@
  k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
  quay.io/ceph/ceph:v16.2.7
  quay.io/cephcsi/cephcsi:v3.5.0
+ quay.io/csiaddons/k8s-sidecar:v0.1.0
  quay.io/csiaddons/volumereplication-operator:v0.1.0
  rook/ceph:master

deploy/examples/operator-openshift.yaml

@@ -403,6 +403,7 @@ data:
   # Configure CSI RBD grpc and liveness metrics port
   # CSI_RBD_GRPC_METRICS_PORT: "9090"
   # CSI_RBD_LIVENESS_METRICS_PORT: "9080"
+  # CSIADDONS_PORT: "9070"
 
   # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
   ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
@@ -417,6 +418,9 @@ data:
   # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15.
   ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15"
   # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
+  # Enable the csi addons sidecar.
+  CSI_ENABLE_CSIADDONS: "false"
+  # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 ---
 # The deployment for the rook operator
 # OLM: BEGIN OPERATOR DEPLOYMENT

deploy/examples/operator.yaml

@@ -321,6 +321,7 @@ data:
   # Configure CSI RBD grpc and liveness metrics port
   # CSI_RBD_GRPC_METRICS_PORT: "9090"
   # CSI_RBD_LIVENESS_METRICS_PORT: "9080"
+  # CSIADDONS_PORT: "9070"
 
   # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
   ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
@@ -335,6 +336,9 @@ data:
   # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring
   CSI_ENABLE_VOLUME_REPLICATION: "false"
   # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
+  # Enable the csi addons sidecar.
+  CSI_ENABLE_CSIADDONS: "false"
+  # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 ---
 # OLM: BEGIN OPERATOR DEPLOYMENT
 apiVersion: apps/v1

pkg/operator/ceph/csi/csi.go

@@ -94,6 +94,7 @@ func (r *ReconcileCSI) setParams() error {
 	CSIParam.SnapshotterImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage)
 	CSIParam.KubeletDirPath = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath)
 	CSIParam.VolumeReplicationImage = k8sutil.GetValue(r.opConfig.Parameters, "CSI_VOLUME_REPLICATION_IMAGE", DefaultVolumeReplicationImage)
+	CSIParam.CSIAddonsImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSIADDONS_IMAGE", DefaultCSIAddonsImage)
 	csiCephFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPHFS_POD_LABELS", "")
 	CSIParam.CSICephFSPodLabels = k8sutil.ParseStringToLabels(csiCephFSPodLabels)
 	csiRBDPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RBD_POD_LABELS", "")

pkg/operator/ceph/csi/spec.go

@@ -51,16 +51,19 @@ type Param struct {
 	PluginPriorityClassName        string
 	ProvisionerPriorityClassName   string
 	VolumeReplicationImage         string
+	CSIAddonsImage                 string
 	EnablePluginSelinuxHostMount   bool
 	EnableCSIHostNetwork           bool
 	EnableOMAPGenerator            bool
 	EnableRBDSnapshotter           bool
 	EnableCephFSSnapshotter        bool
 	EnableVolumeReplicationSideCar bool
+	EnableCSIAddonsSideCar         bool
 	LogLevel                       uint8
 	CephFSGRPCMetricsPort          uint16
 	CephFSLivenessMetricsPort      uint16
 	RBDGRPCMetricsPort             uint16
+	CSIAddonsPort                  uint16
 	RBDLivenessMetricsPort         uint16
 	ProvisionerReplicas            int32
 	CSICephFSPodLabels             map[string]string
@@ -105,6 +108,7 @@ var (
 	DefaultSnapshotterImage       = "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
 	DefaultResizerImage           = "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
 	DefaultVolumeReplicationImage = "quay.io/csiaddons/volumereplication-operator:v0.1.0"
+	DefaultCSIAddonsImage         = "quay.io/csiaddons/k8s-sidecar:v0.1.0"
 
 	// Local package template path for RBD
 	//go:embed template/rbd/csi-rbdplugin.yaml
@@ -162,6 +166,7 @@ const (
 	DefaultCephFSLivenessMerticsPort uint16 = 9081
 	DefaultRBDGRPCMerticsPort        uint16 = 9090
 	DefaultRBDLivenessMerticsPort    uint16 = 9080
+	DefaultCSIAddonsPort             uint16 = 9070
 
 	detectCSIVersionName = "rook-ceph-csi-detect-version"
 	// default log level for csi containers
@@ -270,6 +275,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 	if err != nil {
 		return errors.Wrap(err, "error getting CSI RBD GRPC metrics port.")
 	}
+	tp.CSIAddonsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_PORT", DefaultCSIAddonsPort)
+	if err != nil {
+		return errors.Wrap(err, "failed to get CSI Addons port")
+	}
 	tp.RBDLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort)
 	if err != nil {
 		return errors.Wrap(err, "error getting CSI RBD liveness metrics port.")
@@ -305,6 +314,11 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		tp.EnableVolumeReplicationSideCar = true
 	}
 
+	tp.EnableCSIAddonsSideCar = false
+	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CSIADDONS", "false"), "true") {
+		tp.EnableCSIAddonsSideCar = true
+	}
+
 	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) {
 		tp.CephFSPluginUpdateStrategy = onDelete
 	} else {

pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml

@@ -127,6 +127,49 @@ spec:
             - name: socket-dir
               mountPath: /csi
         {{ end }}
+        {{ if .EnableCSIAddonsSideCar }}
+        - name: csi-addons
+          image: {{ .CSIAddonsImage }}
+          args :
+            - "--node-id=$(NODE_ID)"
+            - "--v={{ .LogLevel }}"
+            - "--csi-addons-address=$(CSIADDONS_ENDPOINT)"
+            - "--controller-ip=$(POD_IP)"
+            - "--controller-port={{ .CSIAddonsPort }}"
+            - "--pod=$(POD_NAME)"
+            - "--namespace=$(POD_NAMESPACE)"
+            - "--pod-uid=$(POD_UID)"
+            - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/pv/"
+          ports:
+            - containerPort: {{ .CSIAddonsPort }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CSIADDONS_ENDPOINT
+              value: unix:///csi/csi-addons.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        {{ end }}
         - name: csi-rbdplugin
           image: {{ .CSIPluginImage }}
           args :
@@ -140,6 +183,9 @@ spec:
             - "--metricsport={{ .RBDGRPCMetricsPort }}"
             - "--metricspath=/metrics"
             - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
+            {{ if .EnableCSIAddonsSideCar }}
+            - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
+            {{ end }}
           env:
             - name: POD_IP
               valueFrom:
@@ -155,6 +201,10 @@ spec:
                   fieldPath: metadata.namespace
             - name: CSI_ENDPOINT
               value: unix:///csi/csi-provisioner.sock
+            {{ if .EnableCSIAddonsSideCar }}
+            - name: CSIADDONS_ENDPOINT
+              value: unix:///csi/csi-addons.sock
+            {{ end }}
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
             - name: socket-dir