ceph: add support in RGW to communicate vault with TLS

From ceph v16.2.6 onwards the vault TLS suppport in RGW was added,
include similar changes for RGW.

Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>

.github/workflows/canary-integration-test.yml

@@ -576,9 +576,8 @@ jobs:
         yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec.yaml
         yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2
         yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
+        yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.cephVersion.image" quay.io/ceph/daemon-base:latest-pacific-devel
         kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
-        yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec.yaml
-        sed -i 's/ver1/ver2/g' tests/manifests/test-object.yaml
         kubectl create -f tests/manifests/test-object.yaml
         kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml
 

Documentation/ceph-object-store-crd.md

@@ -197,8 +197,6 @@ For RGW, please note the following:
 $ vault kv put rook/<mybucketkey> key=$(openssl rand -base64 32) # kv engine
 $ vault write -f transit/keys/<mybucketkey> exportable=true # transit engine
 
-* TLS authentication with custom certs between Vault and RGW are yet to be supported.
-
 ## Deleting a CephObjectStore
 
 During deletion of a CephObjectStore resource, Rook protects against accidental or premature

pkg/apis/ceph.rook.io/v1/security.go

@@ -38,6 +38,9 @@ func (kms *KeyManagementServiceSpec) IsTokenAuthEnabled() bool {
 
 // IsTLSEnabled return KMS TLS details are configured
 func (kms *KeyManagementServiceSpec) IsTLSEnabled() bool {
+	if getParam(kms.ConnectionDetails, api.EnvVaultSkipVerify) == "true" {
+		return false
+	}
 	for _, tlsOption := range VaultTLSConnectionDetails {
 		tlsSecretName := getParam(kms.ConnectionDetails, tlsOption)
 		if tlsSecretName != "" {

pkg/daemon/ceph/osd/kms/volumes.go

@@ -29,9 +29,9 @@ const (
 	vaultKeySecretKeyName    = "key"
 
 	// File names of the Secret value when mapping on the filesystem
-	vaultCAFileName   = "vault.ca"
-	vaultCertFileName = "vault.crt"
-	vaultKeyFileName  = "vault.key"
+	VaultCAFileName   = "vault.ca"
+	VaultCertFileName = "vault.crt"
+	VaultKeyFileName  = "vault.key"
 
 	// File name for token file
 	VaultFileName = "vault.token"
@@ -82,25 +82,36 @@ func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) {
 func tlsSecretPath(tlsOption string) string {
 	switch tlsOption {
 	case api.EnvVaultCACert:
-		return vaultCAFileName
+		return VaultCAFileName
 	case api.EnvVaultClientCert:
-		return vaultCertFileName
+		return VaultCertFileName
 	case api.EnvVaultClientKey:
-		return vaultKeyFileName
+		return VaultKeyFileName
 
 	}
 
 	return ""
 }
 
 // VaultTokenFileVolume save token from secret as volume mount
-func VaultTokenFileVolume(tokenSecretName string) v1.Volume {
-	return v1.Volume{
+func VaultTokenFileVolume(tokenSecretName string, config map[string]string) v1.Volume {
+	mode := int32(0400)
+	secretVolumeProjections := TLSSecretVolumeAndMount(config)
+
+	if tokenSecretName != "" {
+		projectionSecret := &v1.SecretProjection{Items: []v1.KeyToPath{{Key: KMSTokenSecretNameKey, Path: VaultFileName, Mode: &mode}}}
+		projectionSecret.Name = tokenSecretName
+		secretProjection := v1.VolumeProjection{Secret: projectionSecret}
+		secretVolumeProjections = append(secretVolumeProjections, secretProjection)
+	}
+	volume := v1.Volume{
 		Name: secrets.TypeVault,
 		VolumeSource: v1.VolumeSource{
-			Secret: &v1.SecretVolumeSource{
-				SecretName: tokenSecretName,
-				Items: []v1.KeyToPath{
-					{Key: KMSTokenSecretNameKey, Path: VaultFileName},
-				}}}}
+			Projected: &v1.ProjectedVolumeSource{
+				Sources: secretVolumeProjections,
+			},
+		},
+	}
+
+	return volume
 }

pkg/daemon/ceph/osd/kms/volumes_test.go

@@ -77,3 +77,43 @@ func TestTLSSecretVolumeAndMount(t *testing.T) {
 		})
 	}
 }
+func TestVaultTokenVolumeAndMount(t *testing.T) {
+	m := int32(0400)
+	type args struct {
+		tokenSecretName string
+		config          map[string]string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []v1.VolumeProjection
+	}{
+		{"empty", args{tokenSecretName: "", config: map[string]string{"foo": "bar"}}, []v1.VolumeProjection{}},
+		{"token file", args{tokenSecretName: "vault-token", config: map[string]string{"foo": "bar"}}, []v1.VolumeProjection{
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}}},
+		},
+		{"token and single ca", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}}, []v1.VolumeProjection{
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}},
+		}},
+		{"token, ca and client cert", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}}, []v1.VolumeProjection{
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}},
+		}},
+		{"token, ca and client cert/key", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}}, []v1.VolumeProjection{
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-key"}, Items: []v1.KeyToPath{{Key: "key", Path: "vault.key", Mode: &m}}, Optional: nil}},
+			{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}},
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			volume := VaultTokenFileVolume(tt.args.tokenSecretName, tt.args.config)
+			if !reflect.DeepEqual(volume.VolumeSource.Projected.Sources, tt.want) {
+				t.Errorf("VaultTokenFileVolume() = %v, want %v", volume.VolumeSource.Projected.Sources, tt.want)
+			}
+		})
+	}
+}

pkg/operator/ceph/object/spec.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rook/rook/pkg/daemon/ceph/osd/kms"
 	cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/controller"
+	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -46,11 +47,11 @@ set -e
 VAULT_TOKEN_OLD_PATH=%s
 VAULT_TOKEN_NEW_PATH=%s
 
-cp --verbose $VAULT_TOKEN_OLD_PATH $VAULT_TOKEN_NEW_PATH
+cp -r --verbose $VAULT_TOKEN_OLD_PATH/..data/. $VAULT_TOKEN_NEW_PATH
 
-chmod --verbose 400 $VAULT_TOKEN_NEW_PATH
+chmod -R --verbose 400 $VAULT_TOKEN_NEW_PATH
 
-chown --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH
+chown -R --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH
 `
 )
 
@@ -158,8 +159,8 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
 	}
 	if kmsEnabled {
 		if c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() {
-			podSpec.Volumes = append(podSpec.Volumes,
-				kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName))
+			vaultFileVol := kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName, c.store.Spec.Security.KeyManagementService.ConnectionDetails)
+			podSpec.Volumes = append(podSpec.Volumes, vaultFileVol)
 			podSpec.InitContainers = append(podSpec.InitContainers,
 				c.vaultTokenInitContainer(rgwConfig))
 		}
@@ -226,7 +227,7 @@ func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig) v1.Contain
 			"/bin/bash",
 			"-c",
 			fmt.Sprintf(setupVaultTokenFile,
-				path.Join(kms.EtcVaultDir, kms.VaultFileName), path.Join(c.DataPathMap.ContainerDataDir, kms.VaultFileName)),
+				kms.EtcVaultDir, c.DataPathMap.ContainerDataDir),
 		},
 		Image: c.clusterSpec.CephVersion.Image,
 		VolumeMounts: append(
@@ -309,6 +310,23 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
 					c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]),
 			)
 		}
+		if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() &&
+			c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 5}) {
+			container.Args = append(container.Args,
+				cephconfig.NewFlag("rgw crypt vault verify ssl", "true"))
+			if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientCert) != "" {
+				container.Args = append(container.Args,
+					cephconfig.NewFlag("rgw crypt vault ssl clientcert", path.Join(c.DataPathMap.ContainerDataDir, kms.VaultCertFileName)))
+			}
+			if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientKey) != "" {
+				container.Args = append(container.Args,
+					cephconfig.NewFlag("rgw crypt vault ssl clientkey", path.Join(c.DataPathMap.ContainerDataDir, kms.VaultKeyFileName)))
+			}
+			if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultCACert) != "" {
+				container.Args = append(container.Args,
+					cephconfig.NewFlag("rgw crypt vault ssl cacert", path.Join(c.DataPathMap.ContainerDataDir, kms.VaultCAFileName)))
+			}
+		}
 	}
 	return container
 }

tests/manifests/test-object.yaml

@@ -18,5 +18,17 @@ spec:
   preservePoolsOnDelete: false
   gateway:
     port: 80
-    # securePort: 443
     instances: 1
+  security:
+    kms:
+      connectionDetails:
+        KMS_PROVIDER: vault
+        VAULT_ADDR: https://vault.default.svc.cluster.local:8200
+        VAULT_BACKEND_PATH: rook/ver2
+        VAULT_BACKEND: "v2"
+        VAULT_SECRET_ENGINE: kv
+        VAULT_SKIP_VERIFY: "false"
+        VAULT_CACERT: vault-ca-cert
+        VAULT_CLIENT_CERT: vault-client-cert
+        VAULT_CLIENT_KEY: vault-client-key
+      tokenSecretName: rook-vault-token

tests/scripts/deploy-validate-vault.sh

@@ -132,9 +132,13 @@ function validate_rgw_token {
   RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-token-file" | cut -f2- -d=)
   VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-prefix" | cut -f2- -d=)
   VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE)
+  VAULT_CACERT_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-ssl-cacert" | cut -f2- -d=)
+  VAULT_CLIENT_CERT_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-ssl-clientcert" | cut -f2- -d=)
+  VAULT_CLIENT_KEY_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep  "rgw-crypt-vault-ssl-clientkey" | cut -f2- -d=)
+
 
   #fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW
-  FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key)
+  FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl  --key "$VAULT_CLIENT_KEY_FILE" --cert "$VAULT_CLIENT_CERT_FILE" --cacert "$VAULT_CACERT_FILE" -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key)
   if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then
     echo "The set key $ENCRYPTION_KEY is different from fetched key $FETCHED_KEY"
     exit 1