Skip to content

Commit

Permalink
Merge pull request #8579 from thotz/vaultsslsupport
Browse files Browse the repository at this point in the history 
ceph: add support in RGW to communicate vault with TLS
  • Loading branch information
@BlaineEXE
BlaineEXE committed Nov 18, 2021
2 parents 3367f7e + aba50d3 commit cd832e5
Show file tree
Hide file tree
Showing 11 changed files with 217 additions and 108 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/canary-integration-test.yml
View file
Expand Up @@ -422,7 +422,7 @@ jobs:
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
sed -i 's/ver1/ver2/g' tests/manifests/test-object.yaml
yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2
kubectl create -f tests/manifests/test-object.yaml
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
Expand Down
2 changes: 1 addition & 1 deletion Documentation/ceph-object-store-crd.md
View file
Expand Up @@ -212,7 +212,7 @@ For RGW, please note the following:
$ vault kv put rook/<mybucketkey> key=$(openssl rand -base64 32) # kv engine
$ vault write -f transit/keys/<mybucketkey> exportable=true # transit engine

* TLS authentication with custom certs between Vault and RGW are yet to be supported.
* TLS authentication with custom certificates between Vault and CephObjectStore RGWs are supported from ceph v16.2.6 onwards

## Deleting a CephObjectStore

Expand Down
2 changes: 1 addition & 1 deletion pkg/daemon/ceph/osd/kms/envs.go
View file
Expand Up @@ -49,7 +49,7 @@ func vaultTokenEnvVarFromSecret(tokenSecretName string) v1.EnvVar {
}

// vaultTLSEnvVarFromSecret translates TLS env var which are set to k8s secret name to their actual path on the fs once mounted as volume
// See: TLSSecretVolumeAndMount() for more details
// See: VaultSecretVolumeAndMount() for more details
func vaultTLSEnvVarFromSecret(kmsConfig map[string]string) []v1.EnvVar {
vaultTLSEnvVar := []v1.EnvVar{}

Expand Down
41 changes: 17 additions & 24 deletions pkg/daemon/ceph/osd/kms/volumes.go
View file
Expand Up @@ -29,16 +29,16 @@ const (
vaultKeySecretKeyName = "key"

// File names of the Secret value when mapping on the filesystem
vaultCAFileName = "vault.ca"
vaultCertFileName = "vault.crt"
vaultKeyFileName = "vault.key"
VaultCAFileName = "vault.ca"
VaultCertFileName = "vault.crt"
VaultKeyFileName = "vault.key"

// File name for token file
VaultFileName = "vault.token"
)

// TLSSecretVolumeAndMount return the volume and matching volume mount for mounting the secrets into /etc/vault
func TLSSecretVolumeAndMount(config map[string]string) []v1.VolumeProjection {
// VaultSecretVolumeAndMount return the volume and matching volume mount for mounting the vault secrets into /etc/vault
func VaultSecretVolumeAndMount(kmsVaultConfigFiles map[string]string, tokenSecretName string) []v1.VolumeProjection {
// Projection list
secretVolumeProjections := []v1.VolumeProjection{}

Expand All @@ -49,25 +49,30 @@ func TLSSecretVolumeAndMount(config map[string]string) []v1.VolumeProjection {

// Vault TLS Secrets
for _, tlsOption := range cephv1.VaultTLSConnectionDetails {
tlsSecretName := GetParam(config, tlsOption)
tlsSecretName := GetParam(kmsVaultConfigFiles, tlsOption)
if tlsSecretName != "" {
projectionSecret := &v1.SecretProjection{Items: []v1.KeyToPath{{Key: tlsSecretKeyToCheck(tlsOption), Path: tlsSecretPath(tlsOption), Mode: &mode}}}
projectionSecret.Name = tlsSecretName
secretProjection := v1.VolumeProjection{Secret: projectionSecret}
secretVolumeProjections = append(secretVolumeProjections, secretProjection)
}
}

if tokenSecretName != "" {
projectionSecret := &v1.SecretProjection{Items: []v1.KeyToPath{{Key: KMSTokenSecretNameKey, Path: VaultFileName, Mode: &mode}}}
projectionSecret.Name = tokenSecretName
secretProjection := v1.VolumeProjection{Secret: projectionSecret}
secretVolumeProjections = append(secretVolumeProjections, secretProjection)
}
return secretVolumeProjections
}

// VaultVolumeAndMount returns Vault volume and volume mount
func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) {
func VaultVolumeAndMount(kmsVaultConfigFiles map[string]string, tokenSecretName string) (v1.Volume, v1.VolumeMount) {
v := v1.Volume{
Name: secrets.TypeVault,
VolumeSource: v1.VolumeSource{
Projected: &v1.ProjectedVolumeSource{
Sources: TLSSecretVolumeAndMount(config),
Sources: VaultSecretVolumeAndMount(kmsVaultConfigFiles, tokenSecretName),
},
},
}
Expand All @@ -84,25 +89,13 @@ func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) {
func tlsSecretPath(tlsOption string) string {
switch tlsOption {
case api.EnvVaultCACert:
return vaultCAFileName
return VaultCAFileName
case api.EnvVaultClientCert:
return vaultCertFileName
return VaultCertFileName
case api.EnvVaultClientKey:
return vaultKeyFileName
return VaultKeyFileName

}

return ""
}

// VaultTokenFileVolume save token from secret as volume mount
func VaultTokenFileVolume(tokenSecretName string) v1.Volume {
return v1.Volume{
Name: secrets.TypeVault,
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: tokenSecretName,
Items: []v1.KeyToPath{
{Key: KMSTokenSecretNameKey, Path: VaultFileName},
}}}}
}
35 changes: 27 additions & 8 deletions pkg/daemon/ceph/osd/kms/volumes_test.go
View file
Expand Up @@ -45,34 +45,53 @@ func Test_tlsSecretPath(t *testing.T) {
}
}

func TestTLSSecretVolumeAndMount(t *testing.T) {
func TestVaultSecretVolumeAndMount(t *testing.T) {
m := int32(0444)
type args struct {
config map[string]string
config map[string]string
tokenSecretName string
}
tests := []struct {
name string
args args
want []v1.VolumeProjection
}{
{"empty", args{config: map[string]string{"foo": "bar"}}, []v1.VolumeProjection{}},
{"single ca", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}}, []v1.VolumeProjection{
{"empty", args{config: map[string]string{"foo": "bar"}, tokenSecretName: ""}, []v1.VolumeProjection{}},
{"single ca", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}, tokenSecretName: ""}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}}},
},
{"ca and client cert", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}}, []v1.VolumeProjection{
{"ca and client cert", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}, tokenSecretName: ""}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
}},
{"ca and client cert/key", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}}, []v1.VolumeProjection{
{"ca and client cert/key", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}, tokenSecretName: ""}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-key"}, Items: []v1.KeyToPath{{Key: "key", Path: "vault.key", Mode: &m}}, Optional: nil}},
}},
{"token file", args{tokenSecretName: "vault-token", config: map[string]string{"foo": "bar"}}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}}},
},
{"token and ca", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}},
}},
{"token, ca, and client cert", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}},
}},
{"token, ca, and client cert/key", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-key"}, Items: []v1.KeyToPath{{Key: "key", Path: "vault.key", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}},
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := TLSSecretVolumeAndMount(tt.args.config); !reflect.DeepEqual(got, tt.want) {
t.Errorf("TLSSecretVolumeAndMount() = %v, want %v", got, tt.want)
if got := VaultSecretVolumeAndMount(tt.args.config, tt.args.tokenSecretName); !reflect.DeepEqual(got, tt.want) {
t.Errorf("VaultSecretVolumeAndMount() = %v, want %v", got, tt.want)
}
})
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/operator/ceph/cluster/osd/provision_spec.go
View file
Expand Up @@ -123,7 +123,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re
if c.spec.Security.KeyManagementService.IsEnabled() {
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
if kmsProvider == secrets.TypeVault {
volumeTLS, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails)
volumeTLS, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
volumes = append(volumes, volumeTLS)
}
}
Expand Down Expand Up @@ -282,7 +282,7 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
if c.spec.Security.KeyManagementService.IsEnabled() {
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
if kmsProvider == secrets.TypeVault {
_, volumeMountsTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails)
_, volumeMountsTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
volumeMounts = append(volumeMounts, volumeMountsTLS)
envVars = append(envVars, kms.VaultConfigToEnvVar(c.spec)...)
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/operator/ceph/cluster/osd/spec.go
View file
Expand Up @@ -297,7 +297,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
// Somehow when this happens and we try to update a deployment spec it fails with:
// ValidationError(Pod.spec.volumes[7].projected): missing required field "sources"
if c.spec.Security.KeyManagementService.IsEnabled() && c.spec.Security.KeyManagementService.IsTLSEnabled() {
encryptedVol, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails)
encryptedVol, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
volumes = append(volumes, encryptedVol)
}
}
Expand Down Expand Up @@ -899,7 +899,7 @@ func (c *Cluster) getPVCEncryptionOpenInitContainerActivate(mountPath string, os

// Now let's see if there is a TLS config we need to mount as well
if c.spec.Security.KeyManagementService.IsTLSEnabled() {
_, vaultVolMount := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails)
_, vaultVolMount := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, vaultVolMount)
}

Expand Down
2 changes: 2 additions & 0 deletions pkg/operator/ceph/object/config.go
View file
Expand Up @@ -52,6 +52,8 @@ caps osd = "allow rwx"
rgwPortInternalPort int32 = 8080
ServiceServingCertCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
HttpTimeOut = time.Second * 15
rgwVaultVolumeName = "rgw-vault-volume"
rgwVaultDirName = "/etc/vault/rgw/"
)

var (
Expand Down
57 changes: 43 additions & 14 deletions pkg/operator/ceph/object/spec.go
View file
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/osd/kms"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
Expand All @@ -45,11 +46,11 @@ set -e
VAULT_TOKEN_OLD_PATH=%s
VAULT_TOKEN_NEW_PATH=%s
cp --verbose $VAULT_TOKEN_OLD_PATH $VAULT_TOKEN_NEW_PATH
cp --recursive --verbose $VAULT_TOKEN_OLD_PATH/..data/. $VAULT_TOKEN_NEW_PATH
chmod --verbose 400 $VAULT_TOKEN_NEW_PATH
chown --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH
chmod --recursive --verbose 400 $VAULT_TOKEN_NEW_PATH/*
chmod --verbose 700 $VAULT_TOKEN_NEW_PATH
chown --recursive --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH
`
)

Expand Down Expand Up @@ -157,8 +158,16 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
}
if kmsEnabled {
if c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() {
podSpec.Volumes = append(podSpec.Volumes,
kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName))
vaultFileVol, _ := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails,
c.store.Spec.Security.KeyManagementService.TokenSecretName)
tmpvolume := v1.Volume{
Name: rgwVaultVolumeName,
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
}

podSpec.Volumes = append(podSpec.Volumes, vaultFileVol, tmpvolume)
podSpec.InitContainers = append(podSpec.InitContainers,
c.vaultTokenInitContainer(rgwConfig))
}
Expand Down Expand Up @@ -211,25 +220,26 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig)
}

// The vault token is passed as Secret for rgw container. So it is mounted as read only.
// RGW has restrictions over vault token file, it should owned by same user(ceph) which
// RGW has restrictions over vault token file, it should owned by same user (ceph) which
// rgw daemon runs and all other permission should be nil or zero. Here ownership can be
// changed with help of FSGroup but in openshift environments for security reasons it has
// predefined value, so it won't work there. Hence the token file is copied to containerDataDir
// from mounted secret then ownership/permissions are changed accordingly with help of a
// init container.
// predefined value, so it won't work there. Hence the token file and certs (if present)
// are copied to other volume from mounted secrets then ownership/permissions are changed
// accordingly with help of an init container.
func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig) v1.Container {
_, volMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails)
_, srcVaultVolMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails, "")
tmpVaultMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName}
return v1.Container{
Name: "vault-initcontainer-token-file-setup",
Command: []string{
"/bin/bash",
"-c",
fmt.Sprintf(setupVaultTokenFile,
path.Join(kms.EtcVaultDir, kms.VaultFileName), path.Join(c.DataPathMap.ContainerDataDir, kms.VaultFileName)),
kms.EtcVaultDir, rgwVaultDirName),
},
Image: c.clusterSpec.CephVersion.Image,
VolumeMounts: append(
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName), volMount),
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName), srcVaultVolMount, tmpVaultMount),
Resources: c.store.Spec.Gateway.Resources,
SecurityContext: controller.PodSecurityContext(),
}
Expand Down Expand Up @@ -305,12 +315,31 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault auth", kms.KMSTokenSecretNameKey),
cephconfig.NewFlag("rgw crypt vault token file",
path.Join(c.DataPathMap.ContainerDataDir, kms.VaultFileName)),
path.Join(rgwVaultDirName, kms.VaultFileName)),
cephconfig.NewFlag("rgw crypt vault prefix", c.vaultPrefixRGW()),
cephconfig.NewFlag("rgw crypt vault secret engine",
c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]),
)
}
if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() &&
c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 6}) {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault verify ssl", "true"))
if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientCert) != "" {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault ssl clientcert", path.Join(rgwVaultDirName, kms.VaultCertFileName)))
}
if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientKey) != "" {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault ssl clientkey", path.Join(rgwVaultDirName, kms.VaultKeyFileName)))
}
if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultCACert) != "" {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault ssl cacert", path.Join(rgwVaultDirName, kms.VaultCAFileName)))
}
}
vaultVolMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName}
container.VolumeMounts = append(container.VolumeMounts, vaultVolMount)
}
return container
}
Expand Down

0 comments on commit cd832e5

Please sign in to comment.