diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 964d12e0124d0..529ddc8b85577 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -576,9 +576,8 @@ jobs: yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi + yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.cephVersion.image" quay.io/ceph/daemon-base:latest-pacific-devel kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml - yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec.yaml - sed -i 's/ver1/ver2/g' tests/manifests/test-object.yaml kubectl create -f tests/manifests/test-object.yaml kubectl create -f cluster/examples/kubernetes/ceph/toolbox.yaml diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index ee7c40fca737d..6f39f75842881 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -197,8 +197,6 @@ For RGW, please note the following: $ vault kv put rook/ key=$(openssl rand -base64 32) # kv engine $ vault write -f transit/keys/ exportable=true # transit engine -* TLS authentication with custom certs between Vault and RGW are yet to be supported. - ## Deleting a CephObjectStore During deletion of a CephObjectStore resource, Rook protects against accidental or premature diff --git a/pkg/apis/ceph.rook.io/v1/security.go b/pkg/apis/ceph.rook.io/v1/security.go index 013fd6bcb461e..309235b50a8fb 100644 --- a/pkg/apis/ceph.rook.io/v1/security.go +++ b/pkg/apis/ceph.rook.io/v1/security.go @@ -38,6 +38,9 @@ func (kms *KeyManagementServiceSpec) IsTokenAuthEnabled() bool { // IsTLSEnabled return KMS TLS details are configured func (kms *KeyManagementServiceSpec) IsTLSEnabled() bool { + if getParam(kms.ConnectionDetails, api.EnvVaultSkipVerify) == "true" { + return false + } for _, tlsOption := range VaultTLSConnectionDetails { tlsSecretName := getParam(kms.ConnectionDetails, tlsOption) if tlsSecretName != "" { diff --git a/pkg/daemon/ceph/osd/kms/volumes.go b/pkg/daemon/ceph/osd/kms/volumes.go index 9075b6af0dc55..88045a21e7ce3 100644 --- a/pkg/daemon/ceph/osd/kms/volumes.go +++ b/pkg/daemon/ceph/osd/kms/volumes.go @@ -29,9 +29,9 @@ const ( vaultKeySecretKeyName = "key" // File names of the Secret value when mapping on the filesystem - vaultCAFileName = "vault.ca" - vaultCertFileName = "vault.crt" - vaultKeyFileName = "vault.key" + VaultCAFileName = "vault.ca" + VaultCertFileName = "vault.crt" + VaultKeyFileName = "vault.key" // File name for token file VaultFileName = "vault.token" @@ -82,11 +82,11 @@ func VaultVolumeAndMount(config map[string]string) (v1.Volume, v1.VolumeMount) { func tlsSecretPath(tlsOption string) string { switch tlsOption { case api.EnvVaultCACert: - return vaultCAFileName + return VaultCAFileName case api.EnvVaultClientCert: - return vaultCertFileName + return VaultCertFileName case api.EnvVaultClientKey: - return vaultKeyFileName + return VaultKeyFileName } @@ -94,13 +94,23 @@ func tlsSecretPath(tlsOption string) string { } // VaultTokenFileVolume save token from secret as volume mount -func VaultTokenFileVolume(tokenSecretName string) v1.Volume { - return v1.Volume{ +func VaultTokenFileVolume(tokenSecretName string, config map[string]string) v1.Volume { + mode := int32(0400) + secretVolumeProjections := TLSSecretVolumeAndMount(config) + + projectionSecret := &v1.SecretProjection{Items: []v1.KeyToPath{{Key: KMSTokenSecretNameKey, Path: VaultFileName, Mode: &mode}}} + projectionSecret.Name = tokenSecretName + secretProjection := v1.VolumeProjection{Secret: projectionSecret} + secretVolumeProjections = append(secretVolumeProjections, secretProjection) + + volume := v1.Volume{ Name: secrets.TypeVault, VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: tokenSecretName, - Items: []v1.KeyToPath{ - {Key: KMSTokenSecretNameKey, Path: VaultFileName}, - }}}} + Projected: &v1.ProjectedVolumeSource{ + Sources: secretVolumeProjections, + }, + }, + } + + return volume } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 1aef1d5d2aede..f9be38d8b0d25 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -30,6 +30,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/osd/kms" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/controller" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -46,11 +47,11 @@ set -e VAULT_TOKEN_OLD_PATH=%s VAULT_TOKEN_NEW_PATH=%s -cp --verbose $VAULT_TOKEN_OLD_PATH $VAULT_TOKEN_NEW_PATH +cp -r --verbose $VAULT_TOKEN_OLD_PATH/..data/. $VAULT_TOKEN_NEW_PATH -chmod --verbose 400 $VAULT_TOKEN_NEW_PATH +chmod -R --verbose 400 $VAULT_TOKEN_NEW_PATH -chown --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH +chown -R --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH ` ) @@ -158,8 +159,8 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec } if kmsEnabled { if c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() { - podSpec.Volumes = append(podSpec.Volumes, - kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName)) + vaultFileVol := kms.VaultTokenFileVolume(c.store.Spec.Security.KeyManagementService.TokenSecretName, c.store.Spec.Security.KeyManagementService.ConnectionDetails) + podSpec.Volumes = append(podSpec.Volumes, vaultFileVol) podSpec.InitContainers = append(podSpec.InitContainers, c.vaultTokenInitContainer(rgwConfig)) } @@ -226,7 +227,7 @@ func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig) v1.Contain "/bin/bash", "-c", fmt.Sprintf(setupVaultTokenFile, - path.Join(kms.EtcVaultDir, kms.VaultFileName), path.Join(c.DataPathMap.ContainerDataDir, kms.VaultFileName)), + kms.EtcVaultDir, c.DataPathMap.ContainerDataDir), }, Image: c.clusterSpec.CephVersion.Image, VolumeMounts: append( @@ -309,6 +310,25 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]), ) } + if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() && + c.clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 5}) { + /* _, vaultTLSVolMount := kms.VaultVolumeAndMount(c.store.Spec.Security.KeyManagementService.ConnectionDetails) + container.VolumeMounts = append(container.VolumeMounts, vaultTLSVolMount)*/ + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault verify ssl", "true")) + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientCert) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl clientcert", path.Join(c.DataPathMap.ContainerDataDir, kms.VaultCertFileName))) + } + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientKey) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl clientkey", path.Join(c.DataPathMap.ContainerDataDir, kms.VaultKeyFileName))) + } + if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultCACert) != "" { + container.Args = append(container.Args, + cephconfig.NewFlag("rgw crypt vault ssl cacert", path.Join(c.DataPathMap.ContainerDataDir, kms.VaultCAFileName))) + } + } } return container } diff --git a/tests/manifests/test-object.yaml b/tests/manifests/test-object.yaml index d53b5f346e6aa..b6efedc6ee103 100644 --- a/tests/manifests/test-object.yaml +++ b/tests/manifests/test-object.yaml @@ -18,5 +18,16 @@ spec: preservePoolsOnDelete: false gateway: port: 80 - # securePort: 443 instances: 1 + security: + kms: + connectionDetails: + KMS_PROVIDER: vault + VAULT_ADDR: https://vault.default.svc.cluster.local:8200 + VAULT_BACKEND_PATH: rook/ver2 + VAULT_SECRET_ENGINE: kv + VAULT_SKIP_VERIFY: "false" + VAULT_CACERT: vault-ca-cert + VAULT_CLIENT_CERT: vault-client-cert + VAULT_CLIENT_KEY: vault-client-key + tokenSecretName: rook-vault-token diff --git a/tests/scripts/deploy-validate-vault.sh b/tests/scripts/deploy-validate-vault.sh index 3e54399e60035..f6905c79253aa 100755 --- a/tests/scripts/deploy-validate-vault.sh +++ b/tests/scripts/deploy-validate-vault.sh @@ -132,9 +132,13 @@ function validate_rgw_token { RGW_TOKEN_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-token-file" | cut -f2- -d=) VAULT_PATH_PREFIX=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-prefix" | cut -f2- -d=) VAULT_TOKEN=$(kubectl -n rook-ceph exec $RGW_POD -- cat $RGW_TOKEN_FILE) + VAULT_CACERT_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-cacert" | cut -f2- -d=) + VAULT_CLIENT_CERT_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-clientcert" | cut -f2- -d=) + VAULT_CLIENT_KEY_FILE=$(kubectl -n rook-ceph describe pods "$RGW_POD" | grep "rgw-crypt-vault-ssl-clientkey" | cut -f2- -d=) + #fetch key from vault server using token from RGW pod, P.S using -k for curl since custom ssl certs not yet to support in RGW - FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl -k -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) + FETCHED_KEY=$(kubectl -n rook-ceph exec $RGW_POD -- curl --key "$VAULT_CLIENT_KEY_FILE" --cert "$VAULT_CLIENT_CERT_FILE" --cacert "$VAULT_CACERT_FILE" -X GET -H "X-Vault-Token:$VAULT_TOKEN" "$VAULT_SERVER""$VAULT_PATH_PREFIX"/"$RGW_BUCKET_KEY"|jq -r .data.data.key) if [[ "$ENCRYPTION_KEY" != "$FETCHED_KEY" ]]; then echo "The set key $ENCRYPTION_KEY is different from fetched key $FETCHED_KEY" exit 1