From e9c5a80698bc64d3499e29f766605f23b6fe2ee7 Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Thu, 19 Aug 2021 23:40:38 +0530 Subject: [PATCH] ceph: add support in RGW to communicate vault with TLS From ceph v16.2.6 onwards the vault TLS suppport in RGW was added, include similar changes for RGW. Signed-off-by: Jiffin Tony Thottan --- Documentation/ceph-object-store-crd.md | 2 -- pkg/daemon/ceph/osd/kms/volumes.go | 25 ++++++++++++++-------- pkg/operator/ceph/object/spec.go | 29 +++++++++++++++++++++++--- tests/scripts/deploy-validate-vault.sh | 5 ++++- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index ee7c40fca737d..6f39f75842881 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -197,8 +197,6 @@ For RGW, please note the following: $ vault kv put rook/ key=$(openssl rand -base64 32) # kv engine $ vault write -f transit/keys/ exportable=true # transit engine -* TLS authentication with custom certs between Vault and RGW are yet to be supported. - ## Deleting a CephObjectStore During deletion of a CephObjectStore resource, Rook protects against accidental or premature diff --git a/pkg/daemon/ceph/osd/kms/volumes.go b/pkg/daemon/ceph/osd/kms/volumes.go index 9075b6af0dc55..cef57d53eb462 100644 --- a/pkg/daemon/ceph/osd/kms/volumes.go +++ b/pkg/daemon/ceph/osd/kms/volumes.go @@ -29,9 +29,9 @@ const ( vaultKeySecretKeyName = "key" // File names of the Secret value when mapping on the filesystem - vaultCAFileName = "vault.ca" - vaultCertFileName = "vault.crt" - vaultKeyFileName = "vault.key" + VaultCAFileName = "vault.ca" + VaultCertFileName = "vault.crt" + VaultKeyFileName = "vault.key" // File name for token file VaultFileName = "vault.token" @@ -82,11 +82,11 @@ func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) { func tlsSecretPath(tlsOption string) string { switch tlsOption { case api.EnvVaultCACert: - return vaultCAFileName + return VaultCAFileName case api.EnvVaultClientCert: - return vaultCertFileName + return VaultCertFileName case api.EnvVaultClientKey: - return vaultKeyFileName + return VaultKeyFileName } @@ -94,13 +94,20 @@ func tlsSecretPath(tlsOption string) string { } // VaultTokenFileVolume save token from secret as volume mount -func VaultTokenFileVolume(tokenSecretName string) v1.Volume { - return v1.Volume{ - Name: secrets.TypeVault, +func VaultTokenFileVolume(tokenSecretName string) (v1.Volume, v1.VolumeMount) { + v := v1.Volume{ + Name: "vault-file-vol", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ SecretName: tokenSecretName, Items: []v1.KeyToPath{ {Key: KMSTokenSecretNameKey, Path: VaultFileName}, }}}} + m := v1.VolumeMount{ + Name: "vault-file-vol", + ReadOnly: true, + MountPath: EtcVaultDir, + } + + return v, m } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 1aef1d5d2aede..06e93d53a329e 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -30,6 +30,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/osd/kms" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -158,10 +159,15 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec } if kmsEnabled { if c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() { - podSpec.Volumes = append(podSpec.Volumes, - kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName)) + vaultFileVol, _ := kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName) + podSpec.Volumes = append(podSpec.Volumes, vaultFileVol) podSpec.InitContainers = append(podSpec.InitContainers, c.vaultTokenInitContainer(rgwConfig)) + if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() && + c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 6}) { + vaultTLSVol, _ := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails) + podSpec.Volumes = append(podSpec.Volumes, vaultTLSVol) + } } } c.store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec) @@ -219,7 +225,7 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig) // from mounted secret then ownership/permissions are changed accordingly with help of a // init container. func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig) v1.Container { - _, volMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails) + _, volMount := kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName) return v1.Container{ Name: "vault-initcontainer-token-file-setup", Command: []string{ @@ -309,6 +315,23 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]), ) } + if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() && + c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 6}) { + _, vaultTLSVolMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails) + container.VolumeMounts = append(container.VolumeMounts, vaultTLSVolMount) + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientCert) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl clientcert", path.Join(kms.EtcVaultDir, kms.VaultCertFileName))) + } + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientKey) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl clientkey", path.Join(kms.EtcVaultDir, kms.VaultKeyFileName))) + } + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultCACert) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl cacert", path.Join(kms.EtcVaultDir, kms.VaultCAFileName))) + } + } } return container } diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index dea3cad9ba633..6eb87cdab4bee 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -132,7 +132,10 @@ function validate_rgw_token { RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-token-file" | cut -f2- -d=) VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-prefix" | cut -f2- -d=) VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE) - + #VAULT_CACERT=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-cacert" | cut -f2- -d=) + #VAULT_CLIENT_CERT=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-clientcert" | cut -f2- -d=) + #VAULT_CLIENT_KEY=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-clientkey" | cut -f2- -d=) + #fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then