Merge branch 'rook:master' into test-helm-on-multiple-versions

Documentation/helm-ceph-cluster.md

@@ -88,6 +88,7 @@ The `cephBlockPools` array in the values file will define a list of CephBlockPoo
 | `storageClass.parameters`           | See [Block Storage](ceph-block.md) documentation or the helm values.yaml for suitable values                                                                                                                            | see values.yaml  |
 | `storageClass.reclaimPolicy`        | The default [Reclaim Policy](https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy) to apply to PVCs created with this storage class.                                                             | `Delete`         |
 | `storageClass.allowVolumeExpansion` | Whether [volume expansion](https://kubernetes.io/docs/concepts/storage/storage-classes/#allow-volume-expansion) is allowed by default.                                                                                  | `true`           |
+| `storageClass.mountOptions`         | Specifies the mount options for storageClass                                                                                                                                                                            | `[]`             |
 
 ### Ceph File Systems
 
@@ -101,6 +102,7 @@ The `cephFileSystems` array in the values file will define a list of CephFileSys
 | `storageClass.name`          | The name of the storage class                                                                                                                               | `ceph-filesystem` |
 | `storageClass.parameters`    | See [Shared Filesystem](ceph-filesystem.md) documentation or the helm values.yaml for suitable values                                                       | see values.yaml   |
 | `storageClass.reclaimPolicy` | The default [Reclaim Policy](https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy) to apply to PVCs created with this storage class. | `Delete`          |
+| `storageClass.mountOptions`  | Specifies the mount options for storageClass                                                                                                                | `[]`              |
 
 ### Ceph Object Stores
 

Documentation/helm-operator.md

@@ -110,6 +110,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.provisionerReplicas`           | Set replicas for csi provisioner deployment.                                                                                | `2`                                                       |
 | `csi.enableGrpcMetrics`             | Enable Ceph CSI GRPC Metrics.                                                                                               | `false`                                                   |
 | `csi.enableCSIHostNetwork`          | Enable Host Networking for Ceph CSI nodeplugins.                                                                            | `false`                                                   |
+| `csi.enablePluginSelinuxHostMount`  | Enable Host mount for /etc/selinux directory for Ceph CSI nodeplugins.                                                      | `false`                                                   |
 | `csi.provisionerTolerations`        | Array of tolerations in YAML format which will be added to CSI provisioner deployment.                                      | <none>                                                    |
 | `csi.provisionerNodeAffinity`       | The node labels for affinity of the CSI provisioner deployment (***)                                                        | <none>                                                    |
 | `csi.pluginTolerations`             | Array of tolerations in YAML format which will be added to CephCSI plugin DaemonSet                                         | <none>                                                    |

deploy/charts/rook-ceph-cluster/templates/cephblockpool.yaml

@@ -22,5 +22,11 @@ parameters:
 {{ toYaml $blockpool.storageClass.parameters | indent 2 }}
 reclaimPolicy: {{ default "Delete" $blockpool.storageClass.reclaimPolicy }}
 allowVolumeExpansion: {{ default "true" $blockpool.storageClass.allowVolumeExpansion }}
+{{- if $blockpool.storageClass.mountOptions }}
+mountOptions:
+  {{- range $blockpool.storageClass.mountOptions }}
+  - {{ . }}
+  {{- end }}
+{{- end }}
 {{ end }}
 {{ end }}

deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml

@@ -23,5 +23,11 @@ parameters:
 {{ toYaml $filesystem.storageClass.parameters | indent 2 }}
 reclaimPolicy: {{ default "Delete" $filesystem.storageClass.reclaimPolicy }}
 allowVolumeExpansion: {{ default "true" $filesystem.storageClass.allowVolumeExpansion }}
+{{- if $filesystem.storageClass.mountOptions }}
+mountOptions:
+  {{- range $filesystem.storageClass.mountOptions }}
+  - {{ . }}
+  {{- end }}
+{{- end }}
 {{ end }}
 {{ end }}

deploy/charts/rook-ceph-cluster/values.yaml

@@ -329,6 +329,7 @@ cephBlockPools:
       isDefault: true
       reclaimPolicy: Delete
       allowVolumeExpansion: true
+      mountOptions: []
       # see https://github.com/rook/rook/blob/master/Documentation/ceph-block.md#provision-storage for available configuration
       parameters:
         # (optional) mapOptions is a comma-separated list of map options.
@@ -381,6 +382,7 @@ cephFileSystems:
       name: ceph-filesystem
       reclaimPolicy: Delete
       allowVolumeExpansion: true
+      mountOptions: []
       # see https://github.com/rook/rook/blob/master/Documentation/ceph-filesystem.md#provision-storage for available configuration
       parameters:
         # The secrets contain Ceph admin credentials.

deploy/charts/rook-ceph/templates/deployment.yaml

@@ -89,6 +89,8 @@ spec:
           value: {{ .Values.csi.enableOMAPGenerator | quote }}
         - name: CSI_ENABLE_VOLUME_REPLICATION
           value: {{ .Values.csi.volumeReplication.enabled | quote }}
+        - name: CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT
+          value: {{ .Values.csi.enablePluginSelinuxHostMount | quote }}
 {{- if .Values.csi.enableCSIHostNetwork }}
         - name: CSI_ENABLE_HOST_NETWORK
           value: {{ .Values.csi.enableCSIHostNetwork | quote }}

deploy/charts/rook-ceph/values.yaml

@@ -66,6 +66,8 @@ csi:
   enableCephfsSnapshotter: true
   # set to false to disable deployment of snapshotter container in RBD provisioner pod.
   enableRBDSnapshotter: true
+  # set to false if the selinux is not enabled or unavailable in cluster nodes.
+  enablePluginSelinuxHostMount : false
   # (Optional) set user created priorityclassName for csi plugin pods.
   # pluginPriorityClassName: system-node-critical
 

deploy/examples/operator-openshift.yaml

@@ -154,6 +154,10 @@ data:
 
   # (Optional) Allow starting unsupported ceph-csi image
   ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false"
+
+  # (Optional) control the host mount of /etc/selinux for csi plugin pods.
+  CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: "false"
+
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.

deploy/examples/operator.yaml

@@ -72,6 +72,10 @@ data:
 
   # (Optional) Allow starting unsupported ceph-csi image
   ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false"
+
+  # (Optional) control the host mount of /etc/selinux for csi plugin pods.
+  CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: "false"
+
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.

deploy/olm/assemble/metadata-common.yaml

@@ -153,6 +153,16 @@ spec:
         version: v1
         displayName: Ceph Object Store Zone
         description: Represents a Ceph Object Store Zone.
+      - kind: CephBucketNotification
+        name: cephbucketnotifications.ceph.rook.io
+        version: v1
+        displayName: Ceph Bucket Notification
+        description: Represents a Ceph Bucket Notification.
+      - kind: CephBucketTopic
+        name: cephbuckettopics.ceph.rook.io
+        version: v1
+        displayName: Ceph Bucket Topic
+        description: Represents a Ceph Bucket Topic.
   displayName: Rook-Ceph
   description: |
 

deploy/olm/generate-rook-csv.sh

@@ -10,6 +10,7 @@ ASSEMBLE_FILE_K8S="$OLM_CATALOG_DIR/assemble/metadata-k8s.yaml"
 ASSEMBLE_FILE_OCP="$OLM_CATALOG_DIR/assemble/metadata-ocp.yaml"
 ASSEMBLE_FILE_OKD="$OLM_CATALOG_DIR/assemble/metadata-okd.yaml"
 PACKAGE_FILE="$OLM_CATALOG_DIR/assemble/rook-ceph.package.yaml"
+CRDS_FILE="deploy/examples/crds.yaml"
 SUPPORTED_PLATFORMS='k8s|ocp|okd'
 
 operator_sdk="${OPERATOR_SDK:-operator-sdk}"
@@ -262,6 +263,17 @@ function apply_rook_op_img(){
     "${YQ_CMD_WRITE[@]}" "$CSV_FILE_NAME" spec.install.spec.deployments[0].spec.template.spec.containers[0].image "$ROOK_OP_VERSION"
 }
 
+function validate_crds() {
+    crds=$(awk '/Kind:/ {print $2}' $CRDS_FILE | grep -vE "ObjectBucketList|ObjectBucketClaimList" | sed 's/List//' | sort)
+    csv_crds=$(awk '/kind:/ {print $3}' "$CSV_FILE_NAME" | sort)
+    if [ "$crds" != "$csv_crds" ]; then
+        echo "CRDs in $CSV_FILE_NAME do not match CRDs in $CRDS_FILE, see the diff below"
+        echo ""
+        diff <(echo "$crds") <(echo "$csv_crds")
+        exit 1
+    fi
+}
+
 ########
 # MAIN #
 ########

pkg/operator/ceph/csi/spec.go

@@ -51,6 +51,7 @@ type Param struct {
 	PluginPriorityClassName        string
 	ProvisionerPriorityClassName   string
 	VolumeReplicationImage         string
+	EnablePluginSelinuxHostMount   bool
 	EnableCSIHostNetwork           bool
 	EnableOMAPGenerator            bool
 	EnableRBDSnapshotter           bool
@@ -316,6 +317,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		tp.RBDPluginUpdateStrategy = rollingUpdate
 	}
 
+	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT", "false"), "true") {
+		tp.EnablePluginSelinuxHostMount = true
+	}
+
 	logger.Infof("Kubernetes version is %s.%s", ver.Major, ver.Minor)
 
 	tp.ResizerImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage)

pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml

@@ -105,6 +105,11 @@ spec:
               mountPath: /tmp/csi/keys
             - name: host-run-mount
               mountPath: /run/mount
+            {{ if .EnablePluginSelinuxHostMount }}
+            -  name: etc-selinux
+               mountPath: /etc/selinux
+               readOnly: true
+            {{ end }}
         - name: liveness-prometheus
           securityContext:
             privileged: true
@@ -166,3 +171,8 @@ spec:
         - name: host-run-mount
           hostPath:
             path: /run/mount
+        {{ if .EnablePluginSelinuxHostMount }}
+        - name: etc-selinux
+          hostPath:
+            path: /etc/selinux
+        {{ end }}

pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml

@@ -106,6 +106,11 @@ spec:
               mountPath: /tmp/csi/keys
             - name: host-run-mount
               mountPath: /run/mount
+            {{ if .EnablePluginSelinuxHostMount }}
+            - name: etc-selinux
+              mountPath: /etc/selinux
+              readOnly: true
+            {{ end }}
         - name: liveness-prometheus
           securityContext:
             privileged: true
@@ -176,3 +181,8 @@ spec:
         - name: host-run-mount
           hostPath:
             path: /run/mount
+        {{ if .EnablePluginSelinuxHostMount }}
+        - name: etc-selinux
+          hostPath:
+            path: /etc/selinux
+        {{ end }}