ceph: make provisioner replicas configurable

Documentation/helm-operator.md

@@ -108,6 +108,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.rbdFSGroupPolicy`              | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted                                  | ReadWriteOnceWithFSType                                   |
 | `csi.cephFSFSGroupPolicy`           | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted                               | `None`                                                    |
 | `csi.logLevel`                      | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0`                                                       |
+| `csi.provisionerReplicas`           | Set replicas for csi provisioner deployment.                                                                                | `2`                                                       |
 | `csi.enableGrpcMetrics`             | Enable Ceph CSI GRPC Metrics.                                                                                               | `false`                                                   |
 | `csi.enableCSIHostNetwork`          | Enable Host Networking for Ceph CSI nodeplugins.                                                                            | `false`                                                   |
 | `csi.provisionerTolerations`        | Array of tolerations in YAML format which will be added to CSI provisioner deployment.                                      | <none>                                                    |

cluster/charts/rook-ceph/templates/deployment.yaml

@@ -275,6 +275,10 @@ spec:
         - name: CSI_LOG_LEVEL
           value: {{ .Values.csi.logLevel | quote }}
 {{- end }}
+{{- if .Values.csi.provisionerReplicas }}
+        - name: CSI_PROVISIONER_REPLICAS
+          value: {{ .Values.csi.provisionerReplicas | quote }}
+{{- end }}
 {{- if .Values.csi.csiRBDProvisionerResource }}
         - name: CSI_RBD_PROVISIONER_RESOURCE
           value: {{ .Values.csi.csiRBDProvisionerResource | quote }}

cluster/charts/rook-ceph/values.yaml

@@ -87,6 +87,9 @@ csi:
   # sidecar with CSI provisioner pod, to enable set it to true.
   enableOMAPGenerator: false
 
+  # Set replicas for csi provisioner deployment.
+  provisionerReplicas: 2
+
   # Set logging level for csi containers.
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   #logLevel: 0

cluster/examples/kubernetes/ceph/operator-openshift.yaml

@@ -117,6 +117,9 @@ data:
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   # CSI_LOG_LEVEL: "0"
 
+  # Set replicas for csi provisioner deployment.
+  CSI_PROVISIONER_REPLICAS: "2"
+
   # OMAP generator generates the omap mapping between the PV name and the RBD image
   # which helps CSI to identify the rbd images for CSI operations.
   # CSI_ENABLE_OMAP_GENERATOR need to be enabled when we are using rbd mirroring feature.

cluster/examples/kubernetes/ceph/operator.yaml

@@ -41,6 +41,9 @@ data:
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   # CSI_LOG_LEVEL: "0"
 
+  # Set replicas for csi provisioner deployment.
+  CSI_PROVISIONER_REPLICAS: "2"
+
   # OMAP generator will generate the omap mapping between the PV name and the RBD image.
   # CSI_ENABLE_OMAP_GENERATOR need to be enabled when we are using rbd mirroring feature.
   # By default OMAP generator sidecar is deployed with CSI provisioner pod, to disable

pkg/operator/ceph/csi/spec.go

@@ -64,7 +64,7 @@ type Param struct {
 	CephFSLivenessMetricsPort      uint16
 	RBDGRPCMetricsPort             uint16
 	RBDLivenessMetricsPort         uint16
-	ProvisionerReplicas            uint8
+	ProvisionerReplicas            int32
 	CSICephFSPodLabels             map[string]string
 	CSIRBDPodLabels                map[string]string
 }
@@ -173,6 +173,9 @@ const (
 	// default log level for csi containers
 	defaultLogLevel uint8 = 0
 
+	// default provisioner replicas
+	defaultProvisionerReplicas int32 = 2
+
 	// update strategy
 	rollingUpdate = "RollingUpdate"
 	onDelete      = "OnDelete"
@@ -409,14 +412,26 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 		}
 	}
 
-	tp.ProvisionerReplicas = 2
+	tp.ProvisionerReplicas = defaultProvisionerReplicas
 	nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
 	if err == nil {
 		if len(nodes.Items) == 1 {
 			tp.ProvisionerReplicas = 1
+		} else {
+			replicas, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_PROVISIONER_REPLICAS", "2")
+			if err != nil {
+				logger.Warningf("failed to load CSI_PROVISIONER_REPLICAS. Defaulting to %d. %v", tp.ProvisionerReplicas, err)
+			} else {
+				r, err := strconv.ParseInt(replicas, 10, 32)
+				if err != nil {
+					logger.Errorf("failed to parse CSI_PROVISIONER_REPLICAS. Defaulting to %d. %v", tp.ProvisionerReplicas, err)
+				} else {
+					tp.ProvisionerReplicas = int32(r)
+				}
+			}
 		}
 	} else {
-		logger.Errorf("failed to get nodes. Defaulting the number of replicas of provisioner pods to 2. %v", err)
+		logger.Errorf("failed to get nodes. Defaulting the number of replicas of provisioner pods to %d. %v", tp.ProvisionerReplicas, err)
 	}
 
 	if EnableRBD {