ceph: add support in RGW to communicate vault with TLS

From ceph v16.2.6 onwards the vault TLS suppport in RGW was added,
include similar changes for RGW.

Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>

Documentation/ceph-object-store-crd.md

@@ -197,8 +197,6 @@ For RGW, please note the following:
 $ vault kv put rook/<mybucketkey> key=$(openssl rand -base64 32) # kv engine
 $ vault write -f transit/keys/<mybucketkey> exportable=true # transit engine
 
-* TLS authentication with custom certs between Vault and RGW are yet to be supported.
-
 ## Deleting a CephObjectStore
 
 During deletion of a CephObjectStore resource, Rook protects against accidental or premature

pkg/daemon/ceph/osd/kms/volumes.go

@@ -29,9 +29,9 @@ const (
 	vaultKeySecretKeyName    = "key"
 
 	// File names of the Secret value when mapping on the filesystem
-	vaultCAFileName   = "vault.ca"
-	vaultCertFileName = "vault.crt"
-	vaultKeyFileName  = "vault.key"
+	VaultCAFileName   = "vault.ca"
+	VaultCertFileName = "vault.crt"
+	VaultKeyFileName  = "vault.key"
 
 	// File name for token file
 	VaultFileName = "vault.token"
@@ -82,25 +82,32 @@ func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) {
 func tlsSecretPath(tlsOption string) string {
 	switch tlsOption {
 	case api.EnvVaultCACert:
-		return vaultCAFileName
+		return VaultCAFileName
 	case api.EnvVaultClientCert:
-		return vaultCertFileName
+		return VaultCertFileName
 	case api.EnvVaultClientKey:
-		return vaultKeyFileName
+		return VaultKeyFileName
 
 	}
 
 	return ""
 }
 
 // VaultTokenFileVolume save token from secret as volume mount
-func VaultTokenFileVolume(tokenSecretName string) v1.Volume {
-	return v1.Volume{
-		Name: secrets.TypeVault,
+func VaultTokenFileVolume(tokenSecretName string) (v1.Volume, v1.VolumeMount) {
+	v := v1.Volume{
+		Name: "vault-file-vol",
 		VolumeSource: v1.VolumeSource{
 			Secret: &v1.SecretVolumeSource{
 				SecretName: tokenSecretName,
 				Items: []v1.KeyToPath{
 					{Key: KMSTokenSecretNameKey, Path: VaultFileName},
 				}}}}
+	m := v1.VolumeMount{
+		Name:      "vault-file-vol",
+		ReadOnly:  true,
+		MountPath: EtcVaultDir,
+	}
+
+	return v, m
 }

pkg/operator/ceph/object/spec.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rook/rook/pkg/daemon/ceph/osd/kms"
 	cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/controller"
+	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -158,10 +159,15 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
 	}
 	if kmsEnabled {
 		if c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() {
-			podSpec.Volumes = append(podSpec.Volumes,
-				kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName))
+			vaultFileVol, _ := kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName)
+			podSpec.Volumes = append(podSpec.Volumes, vaultFileVol)
 			podSpec.InitContainers = append(podSpec.InitContainers,
 				c.vaultTokenInitContainer(rgwConfig))
+			if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() &&
+				c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 6}) {
+				vaultTLSVol, _ := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails)
+				podSpec.Volumes = append(podSpec.Volumes, vaultTLSVol)
+			}
 		}
 	}
 	c.store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec)
@@ -219,7 +225,7 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig)
 // from mounted secret then ownership/permissions are changed accordingly with help of a
 // init container.
 func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig) v1.Container {
-	_, volMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails)
+	_, volMount := kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName)
 	return v1.Container{
 		Name: "vault-initcontainer-token-file-setup",
 		Command: []string{
@@ -309,6 +315,23 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
 					c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]),
 			)
 		}
+		if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() &&
+			c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 6}) {
+			_, vaultTLSVolMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails)
+			container.VolumeMounts = append(container.VolumeMounts, vaultTLSVolMount)
+			if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientCert) != "" {
+				container.Args = append(container.Args,
+					cephconfig.NewFlag("rgw crypt vault ssl clientcert", path.Join(kms.EtcVaultDir, kms.VaultCertFileName)))
+			}
+			if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientKey) != "" {
+				container.Args = append(container.Args,
+					cephconfig.NewFlag("rgw crypt vault ssl clientkey", path.Join(kms.EtcVaultDir, kms.VaultKeyFileName)))
+			}
+			if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultCACert) != "" {
+				container.Args = append(container.Args,
+					cephconfig.NewFlag("rgw crypt vault ssl cacert", path.Join(kms.EtcVaultDir, kms.VaultCAFileName)))
+			}
+		}
 	}
 	return container
 }

tests/scripts/deploy-validate-vault.sh

@@ -132,7 +132,10 @@ function validate_rgw_token {
   RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-token-file" | cut -f2- -d=)
   VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-prefix" | cut -f2- -d=)
   VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE)
-
+  #VAULT_CACERT=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-ssl-cacert" | cut -f2- -d=)
+  #VAULT_CLIENT_CERT=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-ssl-clientcert" | cut -f2- -d=)
+  #VAULT_CLIENT_KEY=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-ssl-clientkey" | cut -f2- -d=)
+
   #fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW
   FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key)
   if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then