From aba50d3ca95ea50b6ee0f5f225888d1bfd84887b Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Thu, 19 Aug 2021 23:40:38 +0530 Subject: [PATCH] object: add support in RGW to communicate vault with TLS From ceph v16.2.6 onwards the vault TLS suppport in RGW was added, include similar changes for RGW. Signed-off-by: Jiffin Tony Thottan --- .github/workflows/canary-integration-test.yml | 2 +- Documentation/ceph-object-store-crd.md | 2 +- pkg/daemon/ceph/osd/kms/envs.go | 2 +- pkg/daemon/ceph/osd/kms/volumes.go | 41 ++--- pkg/daemon/ceph/osd/kms/volumes_test.go | 35 +++- .../ceph/cluster/osd/provision_spec.go | 4 +- pkg/operator/ceph/cluster/osd/spec.go | 4 +- pkg/operator/ceph/object/config.go | 2 + pkg/operator/ceph/object/spec.go | 57 ++++-- pkg/operator/ceph/object/spec_test.go | 166 ++++++++++++------ tests/scripts/deploy-validate-vault.sh | 10 +- 11 files changed, 217 insertions(+), 108 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index b3a9929fe856..5821161395de 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -422,7 +422,7 @@ jobs: yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml - sed -i 's/ver1/ver2/g' tests/manifests/test-object.yaml + yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2 kubectl create -f tests/manifests/test-object.yaml tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index c1e1422fa554..9495f0f15cf4 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -212,7 +212,7 @@ For RGW, please note the following: $ vault kv put rook/ key=$(openssl rand -base64 32) # kv engine $ vault write -f transit/keys/ exportable=true # transit engine -* TLS authentication with custom certs between Vault and RGW are yet to be supported. +* TLS authentication with custom certificates between Vault and CephObjectStore RGWs are supported from ceph v16.2.6 onwards ## Deleting a CephObjectStore diff --git a/pkg/daemon/ceph/osd/kms/envs.go b/pkg/daemon/ceph/osd/kms/envs.go index 5967869fabc5..028eaf5a318f 100644 --- a/pkg/daemon/ceph/osd/kms/envs.go +++ b/pkg/daemon/ceph/osd/kms/envs.go @@ -49,7 +49,7 @@ func vaultTokenEnvVarFromSecret(tokenSecretName string) v1.EnvVar { } // vaultTLSEnvVarFromSecret translates TLS env var which are set to k8s secret name to their actual path on the fs once mounted as volume -// See: TLSSecretVolumeAndMount() for more details +// See: VaultSecretVolumeAndMount() for more details func vaultTLSEnvVarFromSecret(kmsConfig map[string]string) []v1.EnvVar { vaultTLSEnvVar := []v1.EnvVar{} diff --git a/pkg/daemon/ceph/osd/kms/volumes.go b/pkg/daemon/ceph/osd/kms/volumes.go index 7718ec43098f..f3358d913084 100644 --- a/pkg/daemon/ceph/osd/kms/volumes.go +++ b/pkg/daemon/ceph/osd/kms/volumes.go @@ -29,16 +29,16 @@ const ( vaultKeySecretKeyName = "key" // File names of the Secret value when mapping on the filesystem - vaultCAFileName = "vault.ca" - vaultCertFileName = "vault.crt" - vaultKeyFileName = "vault.key" + VaultCAFileName = "vault.ca" + VaultCertFileName = "vault.crt" + VaultKeyFileName = "vault.key" // File name for token file VaultFileName = "vault.token" ) -// TLSSecretVolumeAndMount return the volume and matching volume mount for mounting the secrets into /etc/vault -func TLSSecretVolumeAndMount(config map[string]string) []v1.VolumeProjection { +// VaultSecretVolumeAndMount return the volume and matching volume mount for mounting the vault secrets into /etc/vault +func VaultSecretVolumeAndMount(kmsVaultConfigFiles map[string]string, tokenSecretName string) []v1.VolumeProjection { // Projection list secretVolumeProjections := []v1.VolumeProjection{} @@ -49,7 +49,7 @@ func TLSSecretVolumeAndMount(config map[string]string) []v1.VolumeProjection { // Vault TLS Secrets for _, tlsOption := range cephv1.VaultTLSConnectionDetails { - tlsSecretName := GetParam(config, tlsOption) + tlsSecretName := GetParam(kmsVaultConfigFiles, tlsOption) if tlsSecretName != "" { projectionSecret := &v1.SecretProjection{Items: []v1.KeyToPath{{Key: tlsSecretKeyToCheck(tlsOption), Path: tlsSecretPath(tlsOption), Mode: &mode}}} projectionSecret.Name = tlsSecretName @@ -57,17 +57,22 @@ func TLSSecretVolumeAndMount(config map[string]string) []v1.VolumeProjection { secretVolumeProjections = append(secretVolumeProjections, secretProjection) } } - + if tokenSecretName != "" { + projectionSecret := &v1.SecretProjection{Items: []v1.KeyToPath{{Key: KMSTokenSecretNameKey, Path: VaultFileName, Mode: &mode}}} + projectionSecret.Name = tokenSecretName + secretProjection := v1.VolumeProjection{Secret: projectionSecret} + secretVolumeProjections = append(secretVolumeProjections, secretProjection) + } return secretVolumeProjections } // VaultVolumeAndMount returns Vault volume and volume mount -func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) { +func VaultVolumeAndMount(kmsVaultConfigFiles map[string]string, tokenSecretName string) (v1.Volume, v1.VolumeMount) { v := v1.Volume{ Name: secrets.TypeVault, VolumeSource: v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ - Sources: TLSSecretVolumeAndMount(config), + Sources: VaultSecretVolumeAndMount(kmsVaultConfigFiles, tokenSecretName), }, }, } @@ -84,25 +89,13 @@ func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) { func tlsSecretPath(tlsOption string) string { switch tlsOption { case api.EnvVaultCACert: - return vaultCAFileName + return VaultCAFileName case api.EnvVaultClientCert: - return vaultCertFileName + return VaultCertFileName case api.EnvVaultClientKey: - return vaultKeyFileName + return VaultKeyFileName } return "" } - -// VaultTokenFileVolume save token from secret as volume mount -func VaultTokenFileVolume(tokenSecretName string) v1.Volume { - return v1.Volume{ - Name: secrets.TypeVault, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: tokenSecretName, - Items: []v1.KeyToPath{ - {Key: KMSTokenSecretNameKey, Path: VaultFileName}, - }}}} -} diff --git a/pkg/daemon/ceph/osd/kms/volumes_test.go b/pkg/daemon/ceph/osd/kms/volumes_test.go index 185fe748180e..6e3465bc7044 100644 --- a/pkg/daemon/ceph/osd/kms/volumes_test.go +++ b/pkg/daemon/ceph/osd/kms/volumes_test.go @@ -45,34 +45,53 @@ func Test_tlsSecretPath(t *testing.T) { } } -func TestTLSSecretVolumeAndMount(t *testing.T) { +func TestVaultSecretVolumeAndMount(t *testing.T) { m := int32(0444) type args struct { - config map[string]string + config map[string]string + tokenSecretName string } tests := []struct { name string args args want []v1.VolumeProjection }{ - {"empty", args{config: map[string]string{"foo": "bar"}}, []v1.VolumeProjection{}}, - {"single ca", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}}, []v1.VolumeProjection{ + {"empty", args{config: map[string]string{"foo": "bar"}, tokenSecretName: ""}, []v1.VolumeProjection{}}, + {"single ca", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}, tokenSecretName: ""}, []v1.VolumeProjection{ {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}}}, }, - {"ca and client cert", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}}, []v1.VolumeProjection{ + {"ca and client cert", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}, tokenSecretName: ""}, []v1.VolumeProjection{ {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}}, {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}}, }}, - {"ca and client cert/key", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}}, []v1.VolumeProjection{ + {"ca and client cert/key", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}, tokenSecretName: ""}, []v1.VolumeProjection{ {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}}, {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}}, {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-key"}, Items: []v1.KeyToPath{{Key: "key", Path: "vault.key", Mode: &m}}, Optional: nil}}, }}, + {"token file", args{tokenSecretName: "vault-token", config: map[string]string{"foo": "bar"}}, []v1.VolumeProjection{ + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}}}, + }, + {"token and ca", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}}, []v1.VolumeProjection{ + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}}, + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}}, + }}, + {"token, ca, and client cert", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}}, []v1.VolumeProjection{ + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}}, + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}}, + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}}, + }}, + {"token, ca, and client cert/key", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}}, []v1.VolumeProjection{ + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}}, + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}}, + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-key"}, Items: []v1.KeyToPath{{Key: "key", Path: "vault.key", Mode: &m}}, Optional: nil}}, + {Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}}, + }}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := TLSSecretVolumeAndMount(tt.args.config); !reflect.DeepEqual(got, tt.want) { - t.Errorf("TLSSecretVolumeAndMount() = %v, want %v", got, tt.want) + if got := VaultSecretVolumeAndMount(tt.args.config, tt.args.tokenSecretName); !reflect.DeepEqual(got, tt.want) { + t.Errorf("VaultSecretVolumeAndMount() = %v, want %v", got, tt.want) } }) } diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 8d42231426e9..18722fd4d0ee 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -123,7 +123,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re if c.spec.Security.KeyManagementService.IsEnabled() { kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider) if kmsProvider == secrets.TypeVault { - volumeTLS, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails) + volumeTLS, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "") volumes = append(volumes, volumeTLS) } } @@ -282,7 +282,7 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun if c.spec.Security.KeyManagementService.IsEnabled() { kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider) if kmsProvider == secrets.TypeVault { - _, volumeMountsTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails) + _, volumeMountsTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "") volumeMounts = append(volumeMounts, volumeMountsTLS) envVars = append(envVars, kms.VaultConfigToEnvVar(c.spec)...) } diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 7dd339f7a807..3870653cdbd2 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -297,7 +297,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC // Somehow when this happens and we try to update a deployment spec it fails with: // ValidationError(Pod.spec.volumes[7].projected): missing required field "sources" if c.spec.Security.KeyManagementService.IsEnabled() && c.spec.Security.KeyManagementService.IsTLSEnabled() { - encryptedVol, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails) + encryptedVol, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "") volumes = append(volumes, encryptedVol) } } @@ -896,7 +896,7 @@ func (c *Cluster) getPVCEncryptionOpenInitContainerActivate(mountPath string, os // Now let's see if there is a TLS config we need to mount as well if c.spec.Security.KeyManagementService.IsTLSEnabled() { - _, vaultVolMount := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails) + _, vaultVolMount := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "") getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, vaultVolMount) } diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index 6f2456e18e4e..0469998b3677 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -52,6 +52,8 @@ caps osd = "allow rwx" rgwPortInternalPort int32 = 8080 ServiceServingCertCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" HttpTimeOut = time.Second * 15 + rgwVaultVolumeName = "rgw-vault-volume" + rgwVaultDirName = "/etc/vault/rgw/" ) var ( diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 065a813fa762..ee979ed69160 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -29,6 +29,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/osd/kms" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -45,11 +46,11 @@ set -e VAULT_TOKEN_OLD_PATH=%s VAULT_TOKEN_NEW_PATH=%s -cp --verbose $VAULT_TOKEN_OLD_PATH $VAULT_TOKEN_NEW_PATH +cp --recursive --verbose $VAULT_TOKEN_OLD_PATH/..data/. $VAULT_TOKEN_NEW_PATH -chmod --verbose 400 $VAULT_TOKEN_NEW_PATH - -chown --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH +chmod --recursive --verbose 400 $VAULT_TOKEN_NEW_PATH/* +chmod --verbose 700 $VAULT_TOKEN_NEW_PATH +chown --recursive --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH ` ) @@ -157,8 +158,16 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec } if kmsEnabled { if c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() { - podSpec.Volumes = append(podSpec.Volumes, - kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName)) + vaultFileVol, _ := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails, + c.store.Spec.Security.KeyManagementService.TokenSecretName) + tmpvolume := v1.Volume{ + Name: rgwVaultVolumeName, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + } + + podSpec.Volumes = append(podSpec.Volumes, vaultFileVol, tmpvolume) podSpec.InitContainers = append(podSpec.InitContainers, c.vaultTokenInitContainer(rgwConfig)) } @@ -211,25 +220,26 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig) } // The vault token is passed as Secret for rgw container. So it is mounted as read only. -// RGW has restrictions over vault token file, it should owned by same user(ceph) which +// RGW has restrictions over vault token file, it should owned by same user (ceph) which // rgw daemon runs and all other permission should be nil or zero. Here ownership can be // changed with help of FSGroup but in openshift environments for security reasons it has -// predefined value, so it won't work there. Hence the token file is copied to containerDataDir -// from mounted secret then ownership/permissions are changed accordingly with help of a -// init container. +// predefined value, so it won't work there. Hence the token file and certs (if present) +// are copied to other volume from mounted secrets then ownership/permissions are changed +// accordingly with help of an init container. func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig) v1.Container { - _, volMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails) + _, srcVaultVolMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails, "") + tmpVaultMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName} return v1.Container{ Name: "vault-initcontainer-token-file-setup", Command: []string{ "/bin/bash", "-c", fmt.Sprintf(setupVaultTokenFile, - path.Join(kms.EtcVaultDir, kms.VaultFileName), path.Join(c.DataPathMap.ContainerDataDir, kms.VaultFileName)), + kms.EtcVaultDir, rgwVaultDirName), }, Image: c.clusterSpec.CephVersion.Image, VolumeMounts: append( - controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName), volMount), + controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName), srcVaultVolMount, tmpVaultMount), Resources: c.store.Spec.Gateway.Resources, SecurityContext: controller.PodSecurityContext(), } @@ -305,12 +315,31 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { container.Args = append(container.Args, cephconfig.NewFlag("rgw crypt vault auth", kms.KMSTokenSecretNameKey), cephconfig.NewFlag("rgw crypt vault token file", - path.Join(c.DataPathMap.ContainerDataDir, kms.VaultFileName)), + path.Join(rgwVaultDirName, kms.VaultFileName)), cephconfig.NewFlag("rgw crypt vault prefix", c.vaultPrefixRGW()), cephconfig.NewFlag("rgw crypt vault secret engine", c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]), ) } + if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() && + c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 6}) { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault verify ssl", "true")) + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientCert) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl clientcert", path.Join(rgwVaultDirName, kms.VaultCertFileName))) + } + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientKey) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl clientkey", path.Join(rgwVaultDirName, kms.VaultKeyFileName))) + } + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultCACert) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl cacert", path.Join(rgwVaultDirName, kms.VaultCAFileName))) + } + } + vaultVolMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName} + container.VolumeMounts = append(container.VolumeMounts, vaultVolMount) } return container } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 8d0096e0212e..9208b7ed6bf0 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -384,61 +384,123 @@ func TestDefaultReadinessProbe(t *testing.T) { } func TestCheckRGWKMS(t *testing.T) { - ctx := context.TODO() - // Placeholder - context := &clusterd.Context{Clientset: test.New(t, 3)} - store := simpleStore() - store.Spec.Security = &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}} - c := &clusterConfig{ - context: context, - store: store, + setupTest := func() *clusterConfig { + context := &clusterd.Context{Clientset: test.New(t, 3)} + store := simpleStore() + store.Spec.Security = &cephv1.SecuritySpec{KeyManagementService: cephv1.KeyManagementServiceSpec{ConnectionDetails: map[string]string{}}} + return &clusterConfig{ + context: context, + store: store, + } } - - // without KMS - b, err := c.CheckRGWKMS() - assert.False(t, b) - assert.NoError(t, err) - - // setting KMS configurations - c.store.Spec.Security.KeyManagementService.TokenSecretName = "vault-token" - c.store.Spec.Security.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault" - c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200" - s := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: store.Spec.Security.KeyManagementService.TokenSecretName, - Namespace: store.Namespace, - }, - Data: map[string][]byte{ - "token": []byte("myt-otkenbenvqrev"), - }, + configureKMS := func(c *clusterConfig) { + c.store.Spec.Security.KeyManagementService.TokenSecretName = "vault-token" + c.store.Spec.Security.KeyManagementService.ConnectionDetails["KMS_PROVIDER"] = "vault" + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_ADDR"] = "https://1.1.1.1:8200" + s := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.store.Spec.Security.KeyManagementService.TokenSecretName, + Namespace: c.store.Namespace, + }, + Data: map[string][]byte{ + "token": []byte("myt-otkenbenvqrev"), + }, + } + _, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(context.TODO(), s, metav1.CreateOptions{}) + assert.NoError(t, err) } - _, err = context.Clientset.CoreV1().Secrets(store.Namespace).Create(ctx, s, metav1.CreateOptions{}) - assert.NoError(t, err) - - // no secret engine set, will fail - b, err = c.CheckRGWKMS() - assert.False(t, b) - assert.Error(t, err) - - // kv engine version v1, will fail - c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "kv" - c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_BACKEND"] = "v1" - b, err = c.CheckRGWKMS() - assert.False(t, b) - assert.Error(t, err) - // kv engine version v2, will pass - c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_BACKEND"] = "v2" - b, err = c.CheckRGWKMS() - assert.True(t, b) - assert.NoError(t, err) - - // transit engine, will pass - c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "transit" - c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_BACKEND"] = "" - b, err = c.CheckRGWKMS() - assert.True(t, b) - assert.NoError(t, err) + t.Run("KMS is disabled", func(t *testing.T) { + c := setupTest() + b, err := c.CheckRGWKMS() + assert.False(t, b) + assert.NoError(t, err) + }) + + t.Run("Vault Secret Engine is missing", func(t *testing.T) { + c := setupTest() + configureKMS(c) + b, err := c.CheckRGWKMS() + assert.False(t, b) + assert.Error(t, err) + }) + + t.Run("Vault Secret Engine is kv with v1", func(t *testing.T) { + c := setupTest() + configureKMS(c) + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "kv" + b, err := c.CheckRGWKMS() + assert.False(t, b) + assert.Error(t, err) + }) + + t.Run("Vault Secret Engine is kv with v2", func(t *testing.T) { + c := setupTest() + configureKMS(c) + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "kv" + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_BACKEND"] = "v2" + b, err := c.CheckRGWKMS() + assert.True(t, b) + assert.NoError(t, err) + }) + + t.Run("Vault Secret Engine is transit", func(t *testing.T) { + c := setupTest() + configureKMS(c) + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "transit" + b, err := c.CheckRGWKMS() + assert.True(t, b) + assert.NoError(t, err) + }) + + t.Run("TLS is configured but secrets do not exist", func(t *testing.T) { + c := setupTest() + configureKMS(c) + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "transit" + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret" + b, err := c.CheckRGWKMS() + assert.False(t, b) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection details k8s secret \"vault-ca-secret\"") + }) + + t.Run("TLS secret exists but empty key", func(t *testing.T) { + c := setupTest() + configureKMS(c) + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "transit" + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret" + tlsSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-ca-secret", + Namespace: c.store.Namespace, + }, + } + _, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(context.TODO(), tlsSecret, metav1.CreateOptions{}) + assert.NoError(t, err) + b, err := c.CheckRGWKMS() + assert.False(t, b) + assert.Error(t, err, "") + assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection key \"cert\" for \"VAULT_CACERT\" in k8s secret \"vault-ca-secret\"") + }) + + t.Run("TLS config is valid", func(t *testing.T) { + c := setupTest() + configureKMS(c) + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_SECRET_ENGINE"] = "transit" + c.store.Spec.Security.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret" + tlsSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-ca-secret", + Namespace: c.store.Namespace, + }, + } + tlsSecret.Data = map[string][]byte{"cert": []byte("envnrevbnbvsbjkrtn")} + _, err := c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Create(context.TODO(), tlsSecret, metav1.CreateOptions{}) + assert.NoError(t, err) + b, err := c.CheckRGWKMS() + assert.True(t, b) + assert.NoError(t, err, "") + }) } func TestGetDaemonName(t *testing.T) { diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index 2312d73b4eaf..c2f46691a3ad 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -154,12 +154,16 @@ function validate_rgw_token { ENCRYPTION_KEY=$(openssl rand -base64 32) kubectl exec vault-0 -- vault kv put -ca-cert /vault/userconfig/vault-server-tls/vault.crt rook/ver2/"$RGW_BUCKET_KEY" key="$ENCRYPTION_KEY" RGW_POD=$(kubectl -n rook-ceph get pods -l app=rook-ceph-rgw | awk 'FNR == 2 {print $1}') - RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-token-file" | cut -f2- -d=) - VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-prefix" | cut -f2- -d=) + RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-token-file" | cut -f2- -d=) + VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-prefix" | cut -f2- -d=) VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE) + VAULT_CACERT_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-cacert" | cut -f2- -d=) + VAULT_CLIENT_CERT_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-clientcert" | cut -f2- -d=) + VAULT_CLIENT_KEY_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-clientkey" | cut -f2- -d=) + #fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW - FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) + FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl --key "$VAULT_CLIENT_KEY_FILE" --cert "$VAULT_CLIENT_CERT_FILE" --cacert "$VAULT_CACERT_FILE" -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then echo "The set key $ENCRYPTION_KEY is different from fetched key $FETCHED_KEY" exit 1