From e5f5d9be9f6ec8705e06c24a3f58ed5273572d43 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Tue, 17 Aug 2021 09:50:54 +0530 Subject: [PATCH 1/3] csi: mount host's /etc/selinux in node plugins This commit introduces a new configuration option for ceph csi driver to enable hostpath mounting of /etc/selinux directory from the cluster node where csi plugin pods are running, which inturn help the csi driver to specify selinux-related mount options like context. Ref# https://github.com/ceph/ceph-csi/issues/2295 The default value for this configuration is true and if cluster nodes are running without selinux enabled, an admin can deploy csi pods by specifying this option to `false` which skip the host path mounting for the csi pods. Signed-off-by: Humble Chirammal --- Documentation/helm-operator.md | 1 + deploy/charts/rook-ceph/templates/deployment.yaml | 2 ++ deploy/charts/rook-ceph/values.yaml | 2 ++ deploy/examples/operator-openshift.yaml | 4 ++++ deploy/examples/operator.yaml | 4 ++++ pkg/operator/ceph/csi/spec.go | 5 +++++ .../ceph/csi/template/cephfs/csi-cephfsplugin.yaml | 10 ++++++++++ pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml | 10 ++++++++++ 8 files changed, 38 insertions(+) diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index 1febdd809b055..5e69ab793db54 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -110,6 +110,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.provisionerReplicas` | Set replicas for csi provisioner deployment. | `2` | | `csi.enableGrpcMetrics` | Enable Ceph CSI GRPC Metrics. | `false` | | `csi.enableCSIHostNetwork` | Enable Host Networking for Ceph CSI nodeplugins. | `false` | +| `csi.enablePluginSelinuxHostMount` | Enable Host mount for /etc/selinux directory for Ceph CSI nodeplugins. | `false` | | `csi.provisionerTolerations` | Array of tolerations in YAML format which will be added to CSI provisioner deployment. | | | `csi.provisionerNodeAffinity` | The node labels for affinity of the CSI provisioner deployment (***) | | | `csi.pluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI plugin DaemonSet | | diff --git a/deploy/charts/rook-ceph/templates/deployment.yaml b/deploy/charts/rook-ceph/templates/deployment.yaml index 9eb1c46077da1..879cb66ed4ae2 100644 --- a/deploy/charts/rook-ceph/templates/deployment.yaml +++ b/deploy/charts/rook-ceph/templates/deployment.yaml @@ -89,6 +89,8 @@ spec: value: {{ .Values.csi.enableOMAPGenerator | quote }} - name: CSI_ENABLE_VOLUME_REPLICATION value: {{ .Values.csi.volumeReplication.enabled | quote }} + - name: CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT + value: {{ .Values.csi.enablePluginSelinuxHostMount | quote }} {{- if .Values.csi.enableCSIHostNetwork }} - name: CSI_ENABLE_HOST_NETWORK value: {{ .Values.csi.enableCSIHostNetwork | quote }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index a13b7bae14444..bca7cfdec08be 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -66,6 +66,8 @@ csi: enableCephfsSnapshotter: true # set to false to disable deployment of snapshotter container in RBD provisioner pod. enableRBDSnapshotter: true + # set to false if the selinux is not enabled or unavailable in cluster nodes. + enablePluginSelinuxHostMount : false # (Optional) set user created priorityclassName for csi plugin pods. # pluginPriorityClassName: system-node-critical diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 6ac93d0f81d2e..e16a75e0f5563 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -154,6 +154,10 @@ data: # (Optional) Allow starting unsupported ceph-csi image ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false" + + # (Optional) control the host mount of /etc/selinux for csi plugin pods. + CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: "false" + # The default version of CSI supported by Rook will be started. To change the version # of the CSI driver to something other than what is officially supported, change # these images to the desired release of the CSI driver. diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 6e2512236e196..dafcd0199a2d9 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -72,6 +72,10 @@ data: # (Optional) Allow starting unsupported ceph-csi image ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false" + + # (Optional) control the host mount of /etc/selinux for csi plugin pods. + CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: "false" + # The default version of CSI supported by Rook will be started. To change the version # of the CSI driver to something other than what is officially supported, change # these images to the desired release of the CSI driver. diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 0acc12017c324..476b39c558c0c 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -51,6 +51,7 @@ type Param struct { PluginPriorityClassName string ProvisionerPriorityClassName string VolumeReplicationImage string + EnablePluginSelinuxHostMount bool EnableCSIHostNetwork bool EnableOMAPGenerator bool EnableRBDSnapshotter bool @@ -316,6 +317,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI tp.RBDPluginUpdateStrategy = rollingUpdate } + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT", "false"), "true") { + tp.EnablePluginSelinuxHostMount = true + } + logger.Infof("Kubernetes version is %s.%s", ver.Major, ver.Minor) tp.ResizerImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage) diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 31251daae52cc..bf9eb76db38cf 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -105,6 +105,11 @@ spec: mountPath: /tmp/csi/keys - name: host-run-mount mountPath: /run/mount + {{ if .EnablePluginSelinuxHostMount }} + - name: etc-selinux + mountPath: /etc/selinux + readOnly: true + {{ end }} - name: liveness-prometheus securityContext: privileged: true @@ -166,3 +171,8 @@ spec: - name: host-run-mount hostPath: path: /run/mount + {{ if .EnablePluginSelinuxHostMount }} + - name: etc-selinux + hostPath: + path: /etc/selinux + {{ end }} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index fe79929f221a1..b6d9767d2e8e9 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -106,6 +106,11 @@ spec: mountPath: /tmp/csi/keys - name: host-run-mount mountPath: /run/mount + {{ if .EnablePluginSelinuxHostMount }} + - name: etc-selinux + mountPath: /etc/selinux + readOnly: true + {{ end }} - name: liveness-prometheus securityContext: privileged: true @@ -176,3 +181,8 @@ spec: - name: host-run-mount hostPath: path: /run/mount + {{ if .EnablePluginSelinuxHostMount }} + - name: etc-selinux + hostPath: + path: /etc/selinux + {{ end }} From caa2c8323b9c06a835b62ea022f9e2376110a1de Mon Sep 17 00:00:00 2001 From: Tom Hellier Date: Wed, 1 Dec 2021 10:50:42 +0000 Subject: [PATCH 2/3] helm: addition of mountOptions into storage class configuration It should be possible to configure the storage classs mount options, this follows the helm code used by the ceph-csi project for their ceph-csi-rbd and ceph-csi-cephfs helm charts. Signed-off-by: Tom Hellier --- Documentation/helm-ceph-cluster.md | 2 ++ .../charts/rook-ceph-cluster/templates/cephblockpool.yaml | 6 ++++++ .../charts/rook-ceph-cluster/templates/cephfilesystem.yaml | 6 ++++++ deploy/charts/rook-ceph-cluster/values.yaml | 2 ++ 4 files changed, 16 insertions(+) diff --git a/Documentation/helm-ceph-cluster.md b/Documentation/helm-ceph-cluster.md index 46a2fb8a793f2..b8167fded0d7d 100644 --- a/Documentation/helm-ceph-cluster.md +++ b/Documentation/helm-ceph-cluster.md @@ -88,6 +88,7 @@ The `cephBlockPools` array in the values file will define a list of CephBlockPoo | `storageClass.parameters` | See [Block Storage](ceph-block.md) documentation or the helm values.yaml for suitable values | see values.yaml | | `storageClass.reclaimPolicy` | The default [Reclaim Policy](https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy) to apply to PVCs created with this storage class. | `Delete` | | `storageClass.allowVolumeExpansion` | Whether [volume expansion](https://kubernetes.io/docs/concepts/storage/storage-classes/#allow-volume-expansion) is allowed by default. | `true` | +| `storageClass.mountOptions` | Specifies the mount options for storageClass | `[]` | ### Ceph File Systems @@ -101,6 +102,7 @@ The `cephFileSystems` array in the values file will define a list of CephFileSys | `storageClass.name` | The name of the storage class | `ceph-filesystem` | | `storageClass.parameters` | See [Shared Filesystem](ceph-filesystem.md) documentation or the helm values.yaml for suitable values | see values.yaml | | `storageClass.reclaimPolicy` | The default [Reclaim Policy](https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy) to apply to PVCs created with this storage class. | `Delete` | +| `storageClass.mountOptions` | Specifies the mount options for storageClass | `[]` | ### Ceph Object Stores diff --git a/deploy/charts/rook-ceph-cluster/templates/cephblockpool.yaml b/deploy/charts/rook-ceph-cluster/templates/cephblockpool.yaml index 41856f5a52876..ad49383808b50 100644 --- a/deploy/charts/rook-ceph-cluster/templates/cephblockpool.yaml +++ b/deploy/charts/rook-ceph-cluster/templates/cephblockpool.yaml @@ -22,5 +22,11 @@ parameters: {{ toYaml $blockpool.storageClass.parameters | indent 2 }} reclaimPolicy: {{ default "Delete" $blockpool.storageClass.reclaimPolicy }} allowVolumeExpansion: {{ default "true" $blockpool.storageClass.allowVolumeExpansion }} +{{- if $blockpool.storageClass.mountOptions }} +mountOptions: + {{- range $blockpool.storageClass.mountOptions }} + - {{ . }} + {{- end }} +{{- end }} {{ end }} {{ end }} diff --git a/deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml b/deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml index 5c5646bef13a5..d2dc1cd12faaf 100644 --- a/deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml +++ b/deploy/charts/rook-ceph-cluster/templates/cephfilesystem.yaml @@ -23,5 +23,11 @@ parameters: {{ toYaml $filesystem.storageClass.parameters | indent 2 }} reclaimPolicy: {{ default "Delete" $filesystem.storageClass.reclaimPolicy }} allowVolumeExpansion: {{ default "true" $filesystem.storageClass.allowVolumeExpansion }} +{{- if $filesystem.storageClass.mountOptions }} +mountOptions: + {{- range $filesystem.storageClass.mountOptions }} + - {{ . }} + {{- end }} +{{- end }} {{ end }} {{ end }} diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index 7edc3aae35b27..91ef523da437f 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -329,6 +329,7 @@ cephBlockPools: isDefault: true reclaimPolicy: Delete allowVolumeExpansion: true + mountOptions: [] # see https://github.com/rook/rook/blob/master/Documentation/ceph-block.md#provision-storage for available configuration parameters: # (optional) mapOptions is a comma-separated list of map options. @@ -381,6 +382,7 @@ cephFileSystems: name: ceph-filesystem reclaimPolicy: Delete allowVolumeExpansion: true + mountOptions: [] # see https://github.com/rook/rook/blob/master/Documentation/ceph-filesystem.md#provision-storage for available configuration parameters: # The secrets contain Ceph admin credentials. From b4a8b01c0ec4bec9cf25ac60b37ebcb5e26051c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 1 Dec 2021 16:48:42 +0100 Subject: [PATCH 3/3] build: add missing topic and notification crd to csv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the missing CRDs as well as a check to prevent missing CRDs from the CSV file. Signed-off-by: Sébastien Han --- deploy/olm/assemble/metadata-common.yaml | 10 ++++++++++ deploy/olm/generate-rook-csv.sh | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/deploy/olm/assemble/metadata-common.yaml b/deploy/olm/assemble/metadata-common.yaml index 34e7c405d0f61..9f4d3a82de832 100644 --- a/deploy/olm/assemble/metadata-common.yaml +++ b/deploy/olm/assemble/metadata-common.yaml @@ -153,6 +153,16 @@ spec: version: v1 displayName: Ceph Object Store Zone description: Represents a Ceph Object Store Zone. + - kind: CephBucketNotification + name: cephbucketnotifications.ceph.rook.io + version: v1 + displayName: Ceph Bucket Notification + description: Represents a Ceph Bucket Notification. + - kind: CephBucketTopic + name: cephbuckettopics.ceph.rook.io + version: v1 + displayName: Ceph Bucket Topic + description: Represents a Ceph Bucket Topic. displayName: Rook-Ceph description: | diff --git a/deploy/olm/generate-rook-csv.sh b/deploy/olm/generate-rook-csv.sh index e59162b91bc12..494624bd5293f 100755 --- a/deploy/olm/generate-rook-csv.sh +++ b/deploy/olm/generate-rook-csv.sh @@ -10,6 +10,7 @@ ASSEMBLE_FILE_K8S="$OLM_CATALOG_DIR/assemble/metadata-k8s.yaml" ASSEMBLE_FILE_OCP="$OLM_CATALOG_DIR/assemble/metadata-ocp.yaml" ASSEMBLE_FILE_OKD="$OLM_CATALOG_DIR/assemble/metadata-okd.yaml" PACKAGE_FILE="$OLM_CATALOG_DIR/assemble/rook-ceph.package.yaml" +CRDS_FILE="deploy/examples/crds.yaml" SUPPORTED_PLATFORMS='k8s|ocp|okd' operator_sdk="${OPERATOR_SDK:-operator-sdk}" @@ -262,6 +263,17 @@ function apply_rook_op_img(){ "${YQ_CMD_WRITE[@]}" "$CSV_FILE_NAME" spec.install.spec.deployments[0].spec.template.spec.containers[0].image "$ROOK_OP_VERSION" } +function validate_crds() { + crds=$(awk '/Kind:/ {print $2}' $CRDS_FILE | grep -vE "ObjectBucketList|ObjectBucketClaimList" | sed 's/List//' | sort) + csv_crds=$(awk '/kind:/ {print $3}' "$CSV_FILE_NAME" | sort) + if [ "$crds" != "$csv_crds" ]; then + echo "CRDs in $CSV_FILE_NAME do not match CRDs in $CRDS_FILE, see the diff below" + echo "" + diff <(echo "$crds") <(echo "$csv_crds") + exit 1 + fi +} + ######## # MAIN # ########