Skip to content

Commit

Permalink
object: add support in RGW to communicate vault with TLS
Browse files Browse the repository at this point in the history 
From ceph v16.2.6 onwards the vault TLS suppport in RGW was added,
include similar changes for RGW.

Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>
  • Loading branch information
@thotz
thotz committed Nov 17, 2021
1 parent bb7306e commit aba50d3
Show file tree
Hide file tree
Showing 11 changed files with 217 additions and 108 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/canary-integration-test.yml
View file
Expand Up @@ -422,7 +422,7 @@ jobs:
yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi
kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml
yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml
sed -i 's/ver1/ver2/g' tests/manifests/test-object.yaml
yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2
kubectl create -f tests/manifests/test-object.yaml
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build cluster/examples/kubernetes/ceph/toolbox.yaml
Expand Down
2 changes: 1 addition & 1 deletion Documentation/ceph-object-store-crd.md
View file
Expand Up @@ -212,7 +212,7 @@ For RGW, please note the following:
$ vault kv put rook/<mybucketkey> key=$(openssl rand -base64 32) # kv engine
$ vault write -f transit/keys/<mybucketkey> exportable=true # transit engine

* TLS authentication with custom certs between Vault and RGW are yet to be supported.
* TLS authentication with custom certificates between Vault and CephObjectStore RGWs are supported from ceph v16.2.6 onwards

## Deleting a CephObjectStore

Expand Down
2 changes: 1 addition & 1 deletion pkg/daemon/ceph/osd/kms/envs.go
View file
Expand Up @@ -49,7 +49,7 @@ func vaultTokenEnvVarFromSecret(tokenSecretName string) v1.EnvVar {
}

// vaultTLSEnvVarFromSecret translates TLS env var which are set to k8s secret name to their actual path on the fs once mounted as volume
// See: TLSSecretVolumeAndMount() for more details
// See: VaultSecretVolumeAndMount() for more details
func vaultTLSEnvVarFromSecret(kmsConfig map[string]string) []v1.EnvVar {
vaultTLSEnvVar := []v1.EnvVar{}

Expand Down
41 changes: 17 additions & 24 deletions pkg/daemon/ceph/osd/kms/volumes.go
View file
Expand Up @@ -29,16 +29,16 @@ const (
vaultKeySecretKeyName = "key"

// File names of the Secret value when mapping on the filesystem
vaultCAFileName = "vault.ca"
vaultCertFileName = "vault.crt"
vaultKeyFileName = "vault.key"
VaultCAFileName = "vault.ca"
VaultCertFileName = "vault.crt"
VaultKeyFileName = "vault.key"

// File name for token file
VaultFileName = "vault.token"
)

// TLSSecretVolumeAndMount return the volume and matching volume mount for mounting the secrets into /etc/vault
func TLSSecretVolumeAndMount(config map[string]string) []v1.VolumeProjection {
// VaultSecretVolumeAndMount return the volume and matching volume mount for mounting the vault secrets into /etc/vault
func VaultSecretVolumeAndMount(kmsVaultConfigFiles map[string]string, tokenSecretName string) []v1.VolumeProjection {
// Projection list
secretVolumeProjections := []v1.VolumeProjection{}

Expand All @@ -49,25 +49,30 @@ func TLSSecretVolumeAndMount(config map[string]string) []v1.VolumeProjection {

// Vault TLS Secrets
for _, tlsOption := range cephv1.VaultTLSConnectionDetails {
tlsSecretName := GetParam(config, tlsOption)
tlsSecretName := GetParam(kmsVaultConfigFiles, tlsOption)
if tlsSecretName != "" {
projectionSecret := &v1.SecretProjection{Items: []v1.KeyToPath{{Key: tlsSecretKeyToCheck(tlsOption), Path: tlsSecretPath(tlsOption), Mode: &mode}}}
projectionSecret.Name = tlsSecretName
secretProjection := v1.VolumeProjection{Secret: projectionSecret}
secretVolumeProjections = append(secretVolumeProjections, secretProjection)
}
}

if tokenSecretName != "" {
projectionSecret := &v1.SecretProjection{Items: []v1.KeyToPath{{Key: KMSTokenSecretNameKey, Path: VaultFileName, Mode: &mode}}}
projectionSecret.Name = tokenSecretName
secretProjection := v1.VolumeProjection{Secret: projectionSecret}
secretVolumeProjections = append(secretVolumeProjections, secretProjection)
}
return secretVolumeProjections
}

// VaultVolumeAndMount returns Vault volume and volume mount
func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) {
func VaultVolumeAndMount(kmsVaultConfigFiles map[string]string, tokenSecretName string) (v1.Volume, v1.VolumeMount) {
v := v1.Volume{
Name: secrets.TypeVault,
VolumeSource: v1.VolumeSource{
Projected: &v1.ProjectedVolumeSource{
Sources: TLSSecretVolumeAndMount(config),
Sources: VaultSecretVolumeAndMount(kmsVaultConfigFiles, tokenSecretName),
},
},
}
Expand All @@ -84,25 +89,13 @@ func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) {
func tlsSecretPath(tlsOption string) string {
switch tlsOption {
case api.EnvVaultCACert:
return vaultCAFileName
return VaultCAFileName
case api.EnvVaultClientCert:
return vaultCertFileName
return VaultCertFileName
case api.EnvVaultClientKey:
return vaultKeyFileName
return VaultKeyFileName

}

return ""
}

// VaultTokenFileVolume save token from secret as volume mount
func VaultTokenFileVolume(tokenSecretName string) v1.Volume {
return v1.Volume{
Name: secrets.TypeVault,
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: tokenSecretName,
Items: []v1.KeyToPath{
{Key: KMSTokenSecretNameKey, Path: VaultFileName},
}}}}
}
35 changes: 27 additions & 8 deletions pkg/daemon/ceph/osd/kms/volumes_test.go
View file
Expand Up @@ -45,34 +45,53 @@ func Test_tlsSecretPath(t *testing.T) {
}
}

func TestTLSSecretVolumeAndMount(t *testing.T) {
func TestVaultSecretVolumeAndMount(t *testing.T) {
m := int32(0444)
type args struct {
config map[string]string
config map[string]string
tokenSecretName string
}
tests := []struct {
name string
args args
want []v1.VolumeProjection
}{
{"empty", args{config: map[string]string{"foo": "bar"}}, []v1.VolumeProjection{}},
{"single ca", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}}, []v1.VolumeProjection{
{"empty", args{config: map[string]string{"foo": "bar"}, tokenSecretName: ""}, []v1.VolumeProjection{}},
{"single ca", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}, tokenSecretName: ""}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}}},
},
{"ca and client cert", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}}, []v1.VolumeProjection{
{"ca and client cert", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}, tokenSecretName: ""}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
}},
{"ca and client cert/key", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}}, []v1.VolumeProjection{
{"ca and client cert/key", args{config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}, tokenSecretName: ""}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-key"}, Items: []v1.KeyToPath{{Key: "key", Path: "vault.key", Mode: &m}}, Optional: nil}},
}},
{"token file", args{tokenSecretName: "vault-token", config: map[string]string{"foo": "bar"}}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}}},
},
{"token and ca", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret"}}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}},
}},
{"token, ca, and client cert", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert"}}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}},
}},
{"token, ca, and client cert/key", args{tokenSecretName: "vault-token", config: map[string]string{"VAULT_CACERT": "vault-ca-secret", "VAULT_CLIENT_CERT": "vault-client-cert", "VAULT_CLIENT_KEY": "vault-client-key"}}, []v1.VolumeProjection{
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-ca-secret"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.ca", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-cert"}, Items: []v1.KeyToPath{{Key: "cert", Path: "vault.crt", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-client-key"}, Items: []v1.KeyToPath{{Key: "key", Path: "vault.key", Mode: &m}}, Optional: nil}},
{Secret: &v1.SecretProjection{LocalObjectReference: v1.LocalObjectReference{Name: "vault-token"}, Items: []v1.KeyToPath{{Key: "token", Path: "vault.token", Mode: &m}}, Optional: nil}},
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := TLSSecretVolumeAndMount(tt.args.config); !reflect.DeepEqual(got, tt.want) {
t.Errorf("TLSSecretVolumeAndMount() = %v, want %v", got, tt.want)
if got := VaultSecretVolumeAndMount(tt.args.config, tt.args.tokenSecretName); !reflect.DeepEqual(got, tt.want) {
t.Errorf("VaultSecretVolumeAndMount() = %v, want %v", got, tt.want)
}
})
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/operator/ceph/cluster/osd/provision_spec.go
View file
Expand Up @@ -123,7 +123,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re
if c.spec.Security.KeyManagementService.IsEnabled() {
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
if kmsProvider == secrets.TypeVault {
volumeTLS, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails)
volumeTLS, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
volumes = append(volumes, volumeTLS)
}
}
Expand Down Expand Up @@ -282,7 +282,7 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun
if c.spec.Security.KeyManagementService.IsEnabled() {
kmsProvider := kms.GetParam(c.spec.Security.KeyManagementService.ConnectionDetails, kms.Provider)
if kmsProvider == secrets.TypeVault {
_, volumeMountsTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails)
_, volumeMountsTLS := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
volumeMounts = append(volumeMounts, volumeMountsTLS)
envVars = append(envVars, kms.VaultConfigToEnvVar(c.spec)...)
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/operator/ceph/cluster/osd/spec.go
View file
Expand Up @@ -297,7 +297,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
// Somehow when this happens and we try to update a deployment spec it fails with:
// ValidationError(Pod.spec.volumes[7].projected): missing required field "sources"
if c.spec.Security.KeyManagementService.IsEnabled() && c.spec.Security.KeyManagementService.IsTLSEnabled() {
encryptedVol, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails)
encryptedVol, _ := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
volumes = append(volumes, encryptedVol)
}
}
Expand Down Expand Up @@ -896,7 +896,7 @@ func (c *Cluster) getPVCEncryptionOpenInitContainerActivate(mountPath string, os

// Now let's see if there is a TLS config we need to mount as well
if c.spec.Security.KeyManagementService.IsTLSEnabled() {
_, vaultVolMount := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails)
_, vaultVolMount := kms.VaultVolumeAndMount(c.spec.Security.KeyManagementService.ConnectionDetails, "")
getKEKFromKMSContainer.VolumeMounts = append(getKEKFromKMSContainer.VolumeMounts, vaultVolMount)
}

Expand Down
2 changes: 2 additions & 0 deletions pkg/operator/ceph/object/config.go
View file
Expand Up @@ -52,6 +52,8 @@ caps osd = "allow rwx"
rgwPortInternalPort int32 = 8080
ServiceServingCertCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
HttpTimeOut = time.Second * 15
rgwVaultVolumeName = "rgw-vault-volume"
rgwVaultDirName = "/etc/vault/rgw/"
)

var (
Expand Down
57 changes: 43 additions & 14 deletions pkg/operator/ceph/object/spec.go
View file
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/osd/kms"
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
Expand All @@ -45,11 +46,11 @@ set -e
VAULT_TOKEN_OLD_PATH=%s
VAULT_TOKEN_NEW_PATH=%s
cp --verbose $VAULT_TOKEN_OLD_PATH $VAULT_TOKEN_NEW_PATH
cp --recursive --verbose $VAULT_TOKEN_OLD_PATH/..data/. $VAULT_TOKEN_NEW_PATH
chmod --verbose 400 $VAULT_TOKEN_NEW_PATH
chown --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH
chmod --recursive --verbose 400 $VAULT_TOKEN_NEW_PATH/*
chmod --verbose 700 $VAULT_TOKEN_NEW_PATH
chown --recursive --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH
`
)

Expand Down Expand Up @@ -157,8 +158,16 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
}
if kmsEnabled {
if c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() {
podSpec.Volumes = append(podSpec.Volumes,
kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName))
vaultFileVol, _ := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails,
c.store.Spec.Security.KeyManagementService.TokenSecretName)
tmpvolume := v1.Volume{
Name: rgwVaultVolumeName,
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
}

podSpec.Volumes = append(podSpec.Volumes, vaultFileVol, tmpvolume)
podSpec.InitContainers = append(podSpec.InitContainers,
c.vaultTokenInitContainer(rgwConfig))
}
Expand Down Expand Up @@ -211,25 +220,26 @@ func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig)
}

// The vault token is passed as Secret for rgw container. So it is mounted as read only.
// RGW has restrictions over vault token file, it should owned by same user(ceph) which
// RGW has restrictions over vault token file, it should owned by same user (ceph) which
// rgw daemon runs and all other permission should be nil or zero. Here ownership can be
// changed with help of FSGroup but in openshift environments for security reasons it has
// predefined value, so it won't work there. Hence the token file is copied to containerDataDir
// from mounted secret then ownership/permissions are changed accordingly with help of a
// init container.
// predefined value, so it won't work there. Hence the token file and certs (if present)
// are copied to other volume from mounted secrets then ownership/permissions are changed
// accordingly with help of an init container.
func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig) v1.Container {
_, volMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails)
_, srcVaultVolMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails, "")
tmpVaultMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName}
return v1.Container{
Name: "vault-initcontainer-token-file-setup",
Command: []string{
"/bin/bash",
"-c",
fmt.Sprintf(setupVaultTokenFile,
path.Join(kms.EtcVaultDir, kms.VaultFileName), path.Join(c.DataPathMap.ContainerDataDir, kms.VaultFileName)),
kms.EtcVaultDir, rgwVaultDirName),
},
Image: c.clusterSpec.CephVersion.Image,
VolumeMounts: append(
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName), volMount),
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName), srcVaultVolMount, tmpVaultMount),
Resources: c.store.Spec.Gateway.Resources,
SecurityContext: controller.PodSecurityContext(),
}
Expand Down Expand Up @@ -305,12 +315,31 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault auth", kms.KMSTokenSecretNameKey),
cephconfig.NewFlag("rgw crypt vault token file",
path.Join(c.DataPathMap.ContainerDataDir, kms.VaultFileName)),
path.Join(rgwVaultDirName, kms.VaultFileName)),
cephconfig.NewFlag("rgw crypt vault prefix", c.vaultPrefixRGW()),
cephconfig.NewFlag("rgw crypt vault secret engine",
c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]),
)
}
if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() &&
c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 6}) {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault verify ssl", "true"))
if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientCert) != "" {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault ssl clientcert", path.Join(rgwVaultDirName, kms.VaultCertFileName)))
}
if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientKey) != "" {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault ssl clientkey", path.Join(rgwVaultDirName, kms.VaultKeyFileName)))
}
if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultCACert) != "" {
container.Args = append(container.Args,
cephconfig.NewFlag("rgw crypt vault ssl cacert", path.Join(rgwVaultDirName, kms.VaultCAFileName)))
}
}
vaultVolMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName}
container.VolumeMounts = append(container.VolumeMounts, vaultVolMount)
}
return container
}
Expand Down

0 comments on commit aba50d3

Please sign in to comment.