From 55e700cc2b6e9c13756be8e69a3fa3b9372c2ec1 Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Thu, 19 Aug 2021 23:40:38 +0530 Subject: [PATCH] ceph: add support in RGW to communicate vault with TLS From ceph v16.2.6 onwards the vault TLS suppport in RGW was added, include similar changes for RGW. Signed-off-by: Jiffin Tony Thottan Signed-off-by: Jiffin Tony Thottan --- .github/workflows/canary-integration-test.yml | 2 ++ Documentation/ceph-object-store-crd.md | 2 -- pkg/apis/ceph.rook.io/v1/security.go | 3 ++ pkg/daemon/ceph/osd/kms/volumes.go | 25 +++++++++------ pkg/operator/ceph/object/spec.go | 31 +++++++++++++++++-- tests/manifests/test-kms-vault-spec.yaml | 3 ++ tests/scripts/deploy-validate-vault.sh | 6 +++- 7 files changed, 57 insertions(+), 15 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 964d12e0124d0..a7c74ddf9249a 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -576,9 +576,11 @@ jobs: yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi + yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.cephVersion.image" quay.io/ceph/daemon-base:latest-pacific-devel kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec.yaml sed -i 's/ver1/ver2/g' tests/manifests/test-object.yaml + sed -i 's/true/false/g' tests/manifests/test-object.yaml kubectl create -f tests/manifests/test-object.yaml kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index ee7c40fca737d..6f39f75842881 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -197,8 +197,6 @@ For RGW, please note the following: $ vault kv put rook/ key=$(openssl rand -base64 32) # kv engine $ vault write -f transit/keys/ exportable=true # transit engine -* TLS authentication with custom certs between Vault and RGW are yet to be supported. - ## Deleting a CephObjectStore During deletion of a CephObjectStore resource, Rook protects against accidental or premature diff --git a/pkg/apis/ceph.rook.io/v1/security.go b/pkg/apis/ceph.rook.io/v1/security.go index 013fd6bcb461e..a1baa9c5e0349 100644 --- a/pkg/apis/ceph.rook.io/v1/security.go +++ b/pkg/apis/ceph.rook.io/v1/security.go @@ -38,6 +38,9 @@ func (kms *KeyManagementServiceSpec) IsTokenAuthEnabled() bool { // IsTLSEnabled return KMS TLS details are configured func (kms *KeyManagementServiceSpec) IsTLSEnabled() bool { + if kms.ConnectDetails[api.EnvVaultSkipVerify] == "true" { + return false + } for _, tlsOption := range VaultTLSConnectionDetails { tlsSecretName := getParam(kms.ConnectionDetails, tlsOption) if tlsSecretName != "" { diff --git a/pkg/daemon/ceph/osd/kms/volumes.go b/pkg/daemon/ceph/osd/kms/volumes.go index 9075b6af0dc55..6e527052544cf 100644 --- a/pkg/daemon/ceph/osd/kms/volumes.go +++ b/pkg/daemon/ceph/osd/kms/volumes.go @@ -29,9 +29,9 @@ const ( vaultKeySecretKeyName = "key" // File names of the Secret value when mapping on the filesystem - vaultCAFileName = "vault.ca" - vaultCertFileName = "vault.crt" - vaultKeyFileName = "vault.key" + VaultCAFileName = "vault.ca" + VaultCertFileName = "vault.crt" + VaultKeyFileName = "vault.key" // File name for token file VaultFileName = "vault.token" @@ -82,11 +82,11 @@ func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) { func tlsSecretPath(tlsOption string) string { switch tlsOption { case api.EnvVaultCACert: - return vaultCAFileName + return VaultCAFileName case api.EnvVaultClientCert: - return vaultCertFileName + return VaultCertFileName case api.EnvVaultClientKey: - return vaultKeyFileName + return VaultKeyFileName } @@ -94,13 +94,20 @@ func tlsSecretPath(tlsOption string) string { } // VaultTokenFileVolume save token from secret as volume mount -func VaultTokenFileVolume(tokenSecretName string) v1.Volume { - return v1.Volume{ - Name: secrets.TypeVault, +func VaultTokenFileVolume(tokenSecretName string) (v1.Volume, v1.VolumeMount) { + volume := v1.Volume{ + Name: "vault-file-vol", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ SecretName: tokenSecretName, Items: []v1.KeyToPath{ {Key: KMSTokenSecretNameKey, Path: VaultFileName}, }}}} + mount := v1.VolumeMount{ + Name: volume.Name, + ReadOnly: true, + MountPath: EtcVaultDir, + } + + return volume, mount } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 1aef1d5d2aede..c1ebe2c496f58 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -30,6 +30,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/osd/kms" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -158,10 +159,15 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec } if kmsEnabled { if c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() { - podSpec.Volumes = append(podSpec.Volumes, - kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName)) + vaultFileVol, _ := kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName) + podSpec.Volumes = append(podSpec.Volumes, vaultFileVol) podSpec.InitContainers = append(podSpec.InitContainers, c.vaultTokenInitContainer(rgwConfig)) + if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() && + c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 5}) { + vaultTLSVol, _ := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails) + podSpec.Volumes = append(podSpec.Volumes, vaultTLSVol) + } } } c.store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec) @@ -219,7 +225,7 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig) // from mounted secret then ownership/permissions are changed accordingly with help of a // init container. func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig) v1.Container { - _, volMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails) + _, volMount := kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName) return v1.Container{ Name: "vault-initcontainer-token-file-setup", Command: []string{ @@ -309,6 +315,25 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]), ) } + if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() && + c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 5}) { + _, vaultTLSVolMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails) + container.VolumeMounts = append(container.VolumeMounts, vaultTLSVolMount) + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault verify ssl", "true")) + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientCert) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl clientcert", path.Join(kms.EtcVaultDir, kms.VaultCertFileName))) + } + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientKey) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl clientkey", path.Join(kms.EtcVaultDir, kms.VaultKeyFileName))) + } + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultCACert) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl cacert", path.Join(kms.EtcVaultDir, kms.VaultCAFileName))) + } + } } return container } diff --git a/tests/manifests/test-kms-vault-spec.yaml b/tests/manifests/test-kms-vault-spec.yaml index d9541f960533b..06caed67138f9 100644 --- a/tests/manifests/test-kms-vault-spec.yaml +++ b/tests/manifests/test-kms-vault-spec.yaml @@ -7,4 +7,7 @@ spec: VAULT_BACKEND_PATH: rook/ver1 VAULT_SECRET_ENGINE: kv VAULT_SKIP_VERIFY: "true" + VAULT_CACERT: vault-ca-cert + VAULT_CLIENT_CERT: vault-client-cert + VAULT_CLIENT_KEY: vault-client-key tokenSecretName: rook-vault-token diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index 3e54399e60035..f6905c79253aa 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -132,9 +132,13 @@ function validate_rgw_token { RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-token-file" | cut -f2- -d=) VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-prefix" | cut -f2- -d=) VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE) + VAULT_CACERT_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-cacert" | cut -f2- -d=) + VAULT_CLIENT_CERT_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-clientcert" | cut -f2- -d=) + VAULT_CLIENT_KEY_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-clientkey" | cut -f2- -d=) + #fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW - FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) + FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl --key "$VAULT_CLIENT_KEY_FILE" --cert "$VAULT_CLIENT_CERT_FILE" --cacert "$VAULT_CACERT_FILE" -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then echo "The set key $ENCRYPTION_KEY is different from fetched key $FETCHED_KEY" exit 1