From e3ca518fec3c5983c04d5a01f420dfbc056ab917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 29 Sep 2021 15:31:23 +0200 Subject: [PATCH] ceph: fix kms auto-detection when full TLS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When TLS is used and includes a caert, client key/cert, we need to copy the content of the secret to a file in the operator's container filesystem so that we can build the TLS config and thus the HTTP Client, which reads those files. Also, removing the files after each API call so they don't persist on the filesystem forever. Signed-off-by: Sébastien Han --- .github/workflows/canary-integration-test.yml | 6 + pkg/daemon/ceph/osd/kms/kms.go | 29 +- pkg/daemon/ceph/osd/kms/kms_test.go | 6 +- pkg/daemon/ceph/osd/kms/vault.go | 50 +++- pkg/daemon/ceph/osd/kms/vault_api.go | 36 ++- pkg/daemon/ceph/osd/kms/vault_api_test.go | 113 +++++++- pkg/daemon/ceph/osd/kms/vault_test.go | 252 +++++++++++------- tests/manifests/test-kms-vault-spec.yaml | 3 + 8 files changed, 375 insertions(+), 120 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 81f1994b636f4..96e8dbb92d5d0 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -140,6 +140,7 @@ jobs: - name: deploy cluster run: | + yq write -d1 -i cluster/examples/kubernetes/ceph/operator.yaml "spec.template.spec.containers[0].image" rook/ceph:local-build kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 @@ -219,6 +220,7 @@ jobs: - name: deploy cluster run: | + yq write -d1 -i cluster/examples/kubernetes/ceph/operator.yaml "spec.template.spec.containers[0].image" rook/ceph:local-build kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -357,6 +359,7 @@ jobs: - name: deploy cluster run: | + yq write -d1 -i cluster/examples/kubernetes/ceph/operator.yaml "spec.template.spec.containers[0].image" rook/ceph:local-build kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi @@ -428,6 +431,7 @@ jobs: - name: deploy cluster run: | + yq write -d1 -i cluster/examples/kubernetes/ceph/operator.yaml "spec.template.spec.containers[0].image" rook/ceph:local-build kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml @@ -571,6 +575,7 @@ jobs: - name: deploy cluster run: | + yq write -d1 -i cluster/examples/kubernetes/ceph/operator.yaml "spec.template.spec.containers[0].image" rook/ceph:local-build kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec.yaml @@ -652,6 +657,7 @@ jobs: - name: deploy cluster run: | + yq write -d1 -i cluster/examples/kubernetes/ceph/operator.yaml "spec.template.spec.containers[0].image" rook/ceph:local-build kubectl create -f cluster/examples/kubernetes/ceph/operator.yaml yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml diff --git a/pkg/daemon/ceph/osd/kms/kms.go b/pkg/daemon/ceph/osd/kms/kms.go index 018143da77cc6..3217d98ac98a9 100644 --- a/pkg/daemon/ceph/osd/kms/kms.go +++ b/pkg/daemon/ceph/osd/kms/kms.go @@ -83,7 +83,14 @@ func (c *Config) PutSecret(secretName, secretValue string) error { } if c.IsVault() { // Store the secret in Vault - v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + removeCertFiles := make(chan bool, 1) + // Remove cert files from operator's filesystem + defer func() { + removeCertFiles <- true + close(removeCertFiles) + }() + + v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails, removeCertFiles) if err != nil { return errors.Wrap(err, "failed to init vault kms") } @@ -102,7 +109,14 @@ func (c *Config) GetSecret(secretName string) (string, error) { var value string if c.IsVault() { // Store the secret in Vault - v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + removeCertFiles := make(chan bool, 1) + // Remove cert files from operator's filesystem + defer func() { + removeCertFiles <- true + close(removeCertFiles) + }() + + v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails, removeCertFiles) if err != nil { return "", errors.Wrap(err, "failed to get secret in vault") } @@ -121,7 +135,14 @@ func (c *Config) GetSecret(secretName string) (string, error) { func (c *Config) DeleteSecret(secretName string) error { if c.IsVault() { // Store the secret in Vault - v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails) + removeCertFiles := make(chan bool, 1) + // Remove cert files from operator's filesystem + defer func() { + removeCertFiles <- true + close(removeCertFiles) + }() + + v, err := InitVault(c.context, c.clusterInfo.Namespace, c.clusterSpec.Security.KeyManagementService.ConnectionDetails, removeCertFiles) if err != nil { return errors.Wrap(err, "failed to delete secret in vault") } @@ -202,7 +223,7 @@ func ValidateConnectionDetails(clusterdContext *clusterd.Context, securitySpec * case VaultKVSecretEngineKey: // Append Backend Version if not already present if GetParam(securitySpec.KeyManagementService.ConnectionDetails, vault.VaultBackendKey) == "" { - backendVersion, err := BackendVersion(securitySpec.KeyManagementService.ConnectionDetails) + backendVersion, err := BackendVersion(clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails) if err != nil { return errors.Wrap(err, "failed to get backend version") } diff --git a/pkg/daemon/ceph/osd/kms/kms_test.go b/pkg/daemon/ceph/osd/kms/kms_test.go index 957f153fea23f..ee2f31cd9f936 100644 --- a/pkg/daemon/ceph/osd/kms/kms_test.go +++ b/pkg/daemon/ceph/osd/kms/kms_test.go @@ -91,7 +91,7 @@ func TestValidateConnectionDetails(t *testing.T) { securitySpec.KeyManagementService.ConnectionDetails["VAULT_CACERT"] = "vault-ca-secret" err = ValidateConnectionDetails(context, securitySpec, ns) assert.Error(t, err, "") - assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection details k8s secret \"VAULT_CACERT\"") + assert.EqualError(t, err, "failed to validate vault connection details: failed to find TLS connection details k8s secret \"vault-ca-secret\"") // Error: TLS secret exists but empty key tlsSecret := &v1.Secret{ @@ -122,7 +122,9 @@ func TestValidateConnectionDetails(t *testing.T) { vault.TestWaitActive(t, core) client := cluster.Cores[0].Client // Mock the client here - vaultClient = func(secretConfig map[string]string) (*api.Client, error) { return client, nil } + vaultClient = func(clusterdContext *clusterd.Context, namespace string, secretConfig map[string]string, removeCertFiles chan bool) (*api.Client, error) { + return client, nil + } if err := client.Sys().Mount("rook/", &api.MountInput{ Type: "kv-v2", Options: map[string]string{"version": "2"}, diff --git a/pkg/daemon/ceph/osd/kms/vault.go b/pkg/daemon/ceph/osd/kms/vault.go index 2c27a3c77f835..801ff0a2d2edb 100644 --- a/pkg/daemon/ceph/osd/kms/vault.go +++ b/pkg/daemon/ceph/osd/kms/vault.go @@ -19,7 +19,9 @@ package kms import ( "context" "io/ioutil" + "os" "strings" + "time" "github.com/hashicorp/vault/api" "github.com/libopenstorage/secrets" @@ -66,7 +68,7 @@ var ( */ // InitVault inits the secret store -func InitVault(context *clusterd.Context, namespace string, config map[string]string) (secrets.Secrets, error) { +func InitVault(context *clusterd.Context, namespace string, config map[string]string, removeCertFiles chan bool) (secrets.Secrets, error) { c := make(map[string]interface{}) // So that we don't alter the content of c.config for later iterations @@ -77,7 +79,7 @@ func InitVault(context *clusterd.Context, namespace string, config map[string]st } // Populate TLS config - newConfigWithTLS, err := configTLS(context, namespace, oriConfig) + newConfigWithTLS, err := configTLS(context, namespace, oriConfig, removeCertFiles) if err != nil { return nil, errors.Wrap(err, "failed to initialize vault tls configuration") } @@ -96,8 +98,9 @@ func InitVault(context *clusterd.Context, namespace string, config map[string]st return v, nil } -func configTLS(clusterdContext *clusterd.Context, namespace string, config map[string]string) (map[string]string, error) { +func configTLS(clusterdContext *clusterd.Context, namespace string, config map[string]string, removeCertFiles chan bool) (map[string]string, error) { ctx := context.TODO() + var filesToRemove []*os.File for _, tlsOption := range cephv1.VaultTLSConnectionDetails { tlsSecretName := GetParam(config, tlsOption) if tlsSecretName == "" { @@ -124,13 +127,50 @@ func configTLS(clusterdContext *clusterd.Context, namespace string, config map[s logger.Debugf("replacing %q current content %q with %q", tlsOption, config[tlsOption], file.Name()) - // update the env var with the path + // Update the env var with the path config[tlsOption] = file.Name() + + // Add the file to the list of files to remove + filesToRemove = append(filesToRemove, file) } else { logger.Debugf("value of tlsOption %q tlsSecretName is already correct %q", tlsOption, tlsSecretName) } } + // Run a goroutine that waits for a confirmation to remove the certificates from the filesystem + go func(filesToRemove []*os.File) { + logger.Debugf("files to remove: %+v", filesToRemove) + rmCertFiles := func() { + for _, file := range filesToRemove { + logger.Debugf("closing %q", file.Name()) + err := file.Close() + if err != nil { + logger.Errorf("failed to close file %q. %v", file.Name(), err) + } + logger.Debugf("closed %q", file.Name()) + logger.Debugf("removing %q", file.Name()) + err = os.Remove(file.Name()) + if err != nil { + logger.Errorf("failed to remove file %q. %v", file.Name(), err) + } + logger.Debugf("removed %q", file.Name()) + } + } + for { + select { + case <-removeCertFiles: + logger.Debug("received confirmation from channel to remove vault certificates") + rmCertFiles() + logger.Info("successfully removed secret files") + return + case <-time.After(time.Second * 5): + logger.Info("never received anything from the channel removing vault cert files anyway") + rmCertFiles() + return + } + } + }(filesToRemove) + return config, nil } @@ -215,7 +255,7 @@ func validateVaultConnectionDetails(clusterdContext *clusterd.Context, ns string // Fetch the secret s, err := clusterdContext.Clientset.CoreV1().Secrets(ns).Get(ctx, tlsSecretName, v1.GetOptions{}) if err != nil { - return errors.Errorf("failed to find TLS connection details k8s secret %q", tlsOption) + return errors.Errorf("failed to find TLS connection details k8s secret %q", tlsSecretName) } // Check the Secret key and its content diff --git a/pkg/daemon/ceph/osd/kms/vault_api.go b/pkg/daemon/ceph/osd/kms/vault_api.go index 2e2cdb2a7788c..a5ceec99afe25 100644 --- a/pkg/daemon/ceph/osd/kms/vault_api.go +++ b/pkg/daemon/ceph/osd/kms/vault_api.go @@ -23,6 +23,7 @@ import ( "github.com/libopenstorage/secrets/vault" "github.com/libopenstorage/secrets/vault/utils" "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" "github.com/hashicorp/vault/api" ) @@ -38,16 +39,34 @@ var vaultClient = newVaultClient // newVaultClient returns a vault client, there is no need for any secretConfig validation // Since this is called after an already validated call InitVault() -func newVaultClient(secretConfig map[string]string) (*api.Client, error) { +func newVaultClient(clusterdContext *clusterd.Context, namespace string, secretConfig map[string]string, removeCertFiles chan bool) (*api.Client, error) { // DefaultConfig uses the environment variables if present. config := api.DefaultConfig() + // Always use a new map otherwise the map will mutate and subsequent calls will fail since the + // TLS content has been altered by the TLS config in vaultClient() + localSecretConfig := make(map[string]string) + for k, v := range secretConfig { + localSecretConfig[k] = v + } + // Convert map string to map interface c := make(map[string]interface{}) - for k, v := range secretConfig { + for k, v := range localSecretConfig { c[k] = v } + // Populate TLS config + newConfigWithTLS, err := configTLS(clusterdContext, namespace, localSecretConfig, removeCertFiles) + if err != nil { + return nil, errors.Wrap(err, "failed to initialize vault tls configuration") + } + + // Populate TLS config + for key, value := range newConfigWithTLS { + c[key] = string(value) + } + // Configure TLS if err := utils.ConfigureTLS(config, c); err != nil { return nil, err @@ -64,7 +83,7 @@ func newVaultClient(secretConfig map[string]string) (*api.Client, error) { client.SetToken(strings.TrimSuffix(os.Getenv(api.EnvVaultToken), "\n")) // Set Vault address, was validated by ValidateConnectionDetails() - err = client.SetAddress(strings.TrimSuffix(secretConfig[api.EnvVaultAddress], "\n")) + err = client.SetAddress(strings.TrimSuffix(localSecretConfig[api.EnvVaultAddress], "\n")) if err != nil { return nil, err } @@ -72,7 +91,7 @@ func newVaultClient(secretConfig map[string]string) (*api.Client, error) { return client, nil } -func BackendVersion(secretConfig map[string]string) (string, error) { +func BackendVersion(clusterdContext *clusterd.Context, namespace string, secretConfig map[string]string) (string, error) { v1 := "v1" v2 := "v2" @@ -90,8 +109,15 @@ func BackendVersion(secretConfig map[string]string) (string, error) { logger.Info("vault kv secret engine version set to v2") return v2, nil default: + removeCertFiles := make(chan bool, 1) + // Remove cert files from operator's filesystem + defer func() { + removeCertFiles <- true + close(removeCertFiles) + }() + // Initialize Vault client - vaultClient, err := vaultClient(secretConfig) + vaultClient, err := vaultClient(clusterdContext, namespace, secretConfig, removeCertFiles) if err != nil { return "", errors.Wrap(err, "failed to initialize vault client") } diff --git a/pkg/daemon/ceph/osd/kms/vault_api_test.go b/pkg/daemon/ceph/osd/kms/vault_api_test.go index 774298271a7eb..cba54a369bb1a 100644 --- a/pkg/daemon/ceph/osd/kms/vault_api_test.go +++ b/pkg/daemon/ceph/osd/kms/vault_api_test.go @@ -17,6 +17,7 @@ limitations under the License. package kms import ( + "context" "testing" kv "github.com/hashicorp/vault-plugin-secrets-kv" @@ -24,6 +25,12 @@ import ( vaulthttp "github.com/hashicorp/vault/http" "github.com/hashicorp/vault/sdk/logical" "github.com/hashicorp/vault/vault" + "github.com/libopenstorage/secrets/vault/utils" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestBackendVersion(t *testing.T) { @@ -35,7 +42,9 @@ func TestBackendVersion(t *testing.T) { client := cluster.Cores[0].Client // Mock the client here - vaultClient = func(secretConfig map[string]string) (*api.Client, error) { return client, nil } + vaultClient = func(clusterdContext *clusterd.Context, namespace string, secretConfig map[string]string, removeCertFiles chan bool) (*api.Client, error) { + return client, nil + } // Set up the kv store if err := client.Sys().Mount("rook/", &api.MountInput{ @@ -67,7 +76,7 @@ func TestBackendVersion(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := BackendVersion(tt.args.secretConfig) + got, err := BackendVersion(&clusterd.Context{}, "ns", tt.args.secretConfig) if (err != nil) != tt.wantErr { t.Errorf("BackendVersion() error = %v, wantErr %v", err, tt.wantErr) return @@ -91,3 +100,103 @@ func fakeVaultServer(t *testing.T) *vault.TestCluster { return cluster } + +func TestTLSConfig(t *testing.T) { + ns := "rook-ceph" + ctx := context.TODO() + context := &clusterd.Context{Clientset: test.New(t, 3)} + secretConfig := map[string]string{ + "foo": "bar", + "KMS_PROVIDER": "vault", + "VAULT_ADDR": "1.1.1.1", + "VAULT_BACKEND_PATH": "vault", + "VAULT_CACERT": "vault-ca-cert", + "VAULT_CLIENT_CERT": "vault-client-cert", + "VAULT_CLIENT_KEY": "vault-client-key", + } + + // DefaultConfig uses the environment variables if present. + config := api.DefaultConfig() + + // Convert map string to map interface + c := make(map[string]interface{}) + for k, v := range secretConfig { + c[k] = v + } + + sCa := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-ca-cert", + Namespace: ns, + }, + Data: map[string][]byte{"cert": []byte(`-----BEGIN CERTIFICATE----- +MIIBJTCB0AIJAPNFNz1CNlDOMA0GCSqGSIb3DQEBCwUAMBoxCzAJBgNVBAYTAkZS +MQswCQYDVQQIDAJGUjAeFw0yMTA5MzAwODAzNDBaFw0yNDA2MjYwODAzNDBaMBox +CzAJBgNVBAYTAkZSMQswCQYDVQQIDAJGUjBcMA0GCSqGSIb3DQEBAQUAA0sAMEgC +QQDHeZ47hVBcryl6SCghM8Zj3Q6DQzJzno1J7EjPXef5m+pIVAEylS9sQuwKtFZc +vv3qS/OVFExmMdbrvfKEIfbBAgMBAAEwDQYJKoZIhvcNAQELBQADQQAAnflLuUM3 +4Dq0v7If4cgae2mr7jj3U/lIpHVtFbF7kVjC/eqmeN1a9u0UbRHKkUr+X1mVX3rJ +BvjQDN6didwQ +-----END CERTIFICATE-----`)}, + } + + sClCert := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-client-cert", + Namespace: ns, + }, + Data: map[string][]byte{"cert": []byte(`-----BEGIN CERTIFICATE----- +MIIBEDCBuwIBATANBgkqhkiG9w0BAQUFADAaMQswCQYDVQQGEwJGUjELMAkGA1UE +CAwCRlIwHhcNMjEwOTMwMDgwNDA1WhcNMjQwNjI2MDgwNDA1WjANMQswCQYDVQQG +EwJGUjBcMA0GCSqGSIb3DQEBAQUAA0sAMEgCQQCpWJqKhSES3BiFkt2M82xy3tkB +plDS8DM0s/+VkqfZlVG18KbbIVDHi1lsPjjs/Aja7lWymw0ycV4KGEcqxdmNAgMB +AAEwDQYJKoZIhvcNAQEFBQADQQC5esmoTqp4uEWyC+GKbTTFp8ngMUywAtZJs4nS +wdoF3ZJJzo4ps0saP1ww5LBdeeXUURscxyaFfCFmGODaHJJn +-----END CERTIFICATE-----`)}, + } + + sClKey := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-client-key", + Namespace: ns, + }, + Data: map[string][]byte{"key": []byte(`-----BEGIN PRIVATE KEY----- +MIIBVgIBADANBgkqhkiG9w0BAQEFAASCAUAwggE8AgEAAkEAqViaioUhEtwYhZLd +jPNsct7ZAaZQ0vAzNLP/lZKn2ZVRtfCm2yFQx4tZbD447PwI2u5VspsNMnFeChhH +KsXZjQIDAQABAkARlCv+oxEq1wQIoZUz83TXe8CFBlGvg9Wc6+5lBWM9F7K4by7i +IB5hQ2oaTNN+1Kxzf+XRM9R7sMPP9qFEp0LhAiEA0PzsQqbvNUVEx8X16Hed6V/Z +yvL1iZeHvc2QIbGjZGkCIQDPcM7U0frsFIPuMY4zpX2b6w4rpxZN7Kybp9/3l0tX +hQIhAJVWVsGeJksLr4WNuRYf+9BbNPdoO/rRNCd2L+tT060ZAiEAl0uontITl9IS +s0yTcZm29lxG9pGkE+uVrOWQ1W0Ud10CIQDJ/L+VCQgjO+SviUECc/nMwhWDMT+V +cjLxGL8tcZjHKg== +-----END PRIVATE KEY-----`)}, + } + + for _, s := range []*v1.Secret{sCa, sClCert, sClKey} { + if _, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, s, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } else { + defer func() { context.Clientset.CoreV1().Secrets(ns).Delete(ctx, s.Name, metav1.DeleteOptions{}) }() + } + } + + removeCertFiles := make(chan bool, 1) + // Remove cert files from operator's filesystem + defer func() { + removeCertFiles <- true + close(removeCertFiles) + }() + + // Populate TLS config + newConfigWithTLS, err := configTLS(context, ns, secretConfig, removeCertFiles) + assert.NoError(t, err) + + // Populate TLS config + for key, value := range newConfigWithTLS { + c[key] = string(value) + } + + // Configure TLS + err = utils.ConfigureTLS(config, c) + assert.NoError(t, err) +} diff --git a/pkg/daemon/ceph/osd/kms/vault_test.go b/pkg/daemon/ceph/osd/kms/vault_test.go index 043462f52edcd..e18321df73788 100644 --- a/pkg/daemon/ceph/osd/kms/vault_test.go +++ b/pkg/daemon/ceph/osd/kms/vault_test.go @@ -19,6 +19,7 @@ package kms import ( "context" "testing" + "time" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/test" @@ -51,111 +52,158 @@ func Test_tlsSecretKeyToCheck(t *testing.T) { func Test_configTLS(t *testing.T) { ctx := context.TODO() - config := map[string]string{ - "foo": "bar", - "KMS_PROVIDER": "vault", - "VAULT_ADDR": "1.1.1.1", - "VAULT_BACKEND_PATH": "vault", - } ns := "rook-ceph" context := &clusterd.Context{Clientset: test.New(t, 3)} - // No tls config - _, err := configTLS(context, ns, config) - assert.NoError(t, err) - - // TLS config with correct values - config = map[string]string{ - "foo": "bar", - "KMS_PROVIDER": "vault", - "VAULT_ADDR": "1.1.1.1", - "VAULT_BACKEND_PATH": "vault", - "VAULT_CACERT": "/etc/vault/cacert", - "VAULT_SKIP_VERIFY": "false", - } - config, err = configTLS(context, ns, config) - assert.NoError(t, err) - assert.Equal(t, "/etc/vault/cacert", config["VAULT_CACERT"]) - - // TLS config but no secret - config = map[string]string{ - "foo": "bar", - "KMS_PROVIDER": "vault", - "VAULT_ADDR": "1.1.1.1", - "VAULT_BACKEND_PATH": "vault", - "VAULT_CACERT": "vault-ca-cert", - "VAULT_SKIP_VERIFY": "false", - } - _, err = configTLS(context, ns, config) - assert.Error(t, err) - assert.EqualError(t, err, "failed to fetch tls k8s secret \"vault-ca-cert\": secrets \"vault-ca-cert\" not found") - - // TLS config success! - config = map[string]string{ - "foo": "bar", - "KMS_PROVIDER": "vault", - "VAULT_ADDR": "1.1.1.1", - "VAULT_BACKEND_PATH": "vault", - "VAULT_CACERT": "vault-ca-cert", - "VAULT_SKIP_VERIFY": "false", - } - s := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vault-ca-cert", - Namespace: ns, - }, - Data: map[string][]byte{"cert": []byte("bar")}, - } - _, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, s, metav1.CreateOptions{}) - assert.NoError(t, err) - config, err = configTLS(context, ns, config) - assert.NoError(t, err) - assert.NotEqual(t, "vault-ca-cert", config["VAULT_CACERT"]) - err = context.Clientset.CoreV1().Secrets(ns).Delete(ctx, s.Name, metav1.DeleteOptions{}) - assert.NoError(t, err) - - // All TLS success! - config = map[string]string{ - "foo": "bar", - "KMS_PROVIDER": "vault", - "VAULT_ADDR": "1.1.1.1", - "VAULT_BACKEND_PATH": "vault", - "VAULT_CACERT": "vault-ca-cert", - "VAULT_CLIENT_CERT": "vault-client-cert", - "VAULT_CLIENT_KEY": "vault-client-key", - } - sCa := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vault-ca-cert", - Namespace: ns, - }, - Data: map[string][]byte{"cert": []byte("bar")}, - } - sClCert := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vault-client-cert", - Namespace: ns, - }, - Data: map[string][]byte{"cert": []byte("bar")}, - } - sClKey := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vault-client-key", - Namespace: ns, - }, - Data: map[string][]byte{"key": []byte("bar")}, - } - _, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, sCa, metav1.CreateOptions{}) - assert.NoError(t, err) - _, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, sClCert, metav1.CreateOptions{}) - assert.NoError(t, err) - _, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, sClKey, metav1.CreateOptions{}) - assert.NoError(t, err) - config, err = configTLS(context, ns, config) - assert.NoError(t, err) - assert.NotEqual(t, "vault-ca-cert", config["VAULT_CACERT"]) - assert.NotEqual(t, "vault-client-cert", config["VAULT_CLIENT_CERT"]) - assert.NotEqual(t, "vault-client-key", config["VAULT_CLIENT_KEY"]) + t.Run("no TLS config", func(t *testing.T) { + config := map[string]string{ + "foo": "bar", + "KMS_PROVIDER": "vault", + "VAULT_ADDR": "1.1.1.1", + "VAULT_BACKEND_PATH": "vault", + } + // No tls config + removeCertFiles := make(chan bool, 1) + _, err := configTLS(context, ns, config, removeCertFiles) + assert.NoError(t, err) + close(removeCertFiles) + }) + + t.Run("TLS config with already populated cert path", func(t *testing.T) { + removeCertFiles := make(chan bool, 1) + config := map[string]string{ + "foo": "bar", + "KMS_PROVIDER": "vault", + "VAULT_ADDR": "1.1.1.1", + "VAULT_BACKEND_PATH": "vault", + "VAULT_CACERT": "/etc/vault/cacert", + "VAULT_SKIP_VERIFY": "false", + } + config, err := configTLS(context, ns, config, removeCertFiles) + assert.NoError(t, err) + assert.Equal(t, "/etc/vault/cacert", config["VAULT_CACERT"]) + close(removeCertFiles) + }) + + t.Run("TLS config but no secret", func(t *testing.T) { + removeCertFiles := make(chan bool, 1) + config := map[string]string{ + "foo": "bar", + "KMS_PROVIDER": "vault", + "VAULT_ADDR": "1.1.1.1", + "VAULT_BACKEND_PATH": "vault", + "VAULT_CACERT": "vault-ca-cert", + "VAULT_SKIP_VERIFY": "false", + } + _, err := configTLS(context, ns, config, removeCertFiles) + assert.Error(t, err) + assert.EqualError(t, err, "failed to fetch tls k8s secret \"vault-ca-cert\": secrets \"vault-ca-cert\" not found") + close(removeCertFiles) + }) + + t.Run("TLS config success!", func(t *testing.T) { + removeCertFiles := make(chan bool, 1) + config := map[string]string{ + "foo": "bar", + "KMS_PROVIDER": "vault", + "VAULT_ADDR": "1.1.1.1", + "VAULT_BACKEND_PATH": "vault", + "VAULT_CACERT": "vault-ca-cert", + "VAULT_SKIP_VERIFY": "false", + } + s := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-ca-cert", + Namespace: ns, + }, + Data: map[string][]byte{"cert": []byte("bar")}, + } + _, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, s, metav1.CreateOptions{}) + assert.NoError(t, err) + config, err = configTLS(context, ns, config, removeCertFiles) + assert.NoError(t, err) + assert.NotEqual(t, "vault-ca-cert", config["VAULT_CACERT"]) + err = context.Clientset.CoreV1().Secrets(ns).Delete(ctx, s.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + removeCertFiles <- true + close(removeCertFiles) + }) + + t.Run("advanced TLS config success!", func(t *testing.T) { + removeCertFiles := make(chan bool, 1) + config := map[string]string{ + "foo": "bar", + "KMS_PROVIDER": "vault", + "VAULT_ADDR": "1.1.1.1", + "VAULT_BACKEND_PATH": "vault", + "VAULT_CACERT": "vault-ca-cert", + "VAULT_CLIENT_CERT": "vault-client-cert", + "VAULT_CLIENT_KEY": "vault-client-key", + } + sCa := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-ca-cert", + Namespace: ns, + }, + Data: map[string][]byte{"cert": []byte("bar")}, + } + sClCert := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-client-cert", + Namespace: ns, + }, + Data: map[string][]byte{"cert": []byte("bar")}, + } + sClKey := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-client-key", + Namespace: ns, + }, + Data: map[string][]byte{"key": []byte("bar")}, + } + _, err := context.Clientset.CoreV1().Secrets(ns).Create(ctx, sCa, metav1.CreateOptions{}) + assert.NoError(t, err) + _, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, sClCert, metav1.CreateOptions{}) + assert.NoError(t, err) + _, err = context.Clientset.CoreV1().Secrets(ns).Create(ctx, sClKey, metav1.CreateOptions{}) + assert.NoError(t, err) + config, err = configTLS(context, ns, config, removeCertFiles) + assert.NoError(t, err) + assert.NotEqual(t, "vault-ca-cert", config["VAULT_CACERT"]) + assert.NotEqual(t, "vault-client-cert", config["VAULT_CLIENT_CERT"]) + assert.NotEqual(t, "vault-client-key", config["VAULT_CLIENT_KEY"]) + assert.FileExists(t, config["VAULT_CACERT"]) + removeCertFiles <- true + close(removeCertFiles) + time.Sleep(time.Second) + assert.NoFileExists(t, config["VAULT_CACERT"]) + assert.NoFileExists(t, config["VAULT_CLIENT_CERT"]) + assert.NoFileExists(t, config["VAULT_CLIENT_KEY"]) + }) + + t.Run("advanced TLS config success with timeout!", func(t *testing.T) { + removeCertFiles := make(chan bool, 1) + config := map[string]string{ + "foo": "bar", + "KMS_PROVIDER": "vault", + "VAULT_ADDR": "1.1.1.1", + "VAULT_BACKEND_PATH": "vault", + "VAULT_CACERT": "vault-ca-cert", + "VAULT_CLIENT_CERT": "vault-client-cert", + "VAULT_CLIENT_KEY": "vault-client-key", + } + config, err := configTLS(context, ns, config, removeCertFiles) + assert.NoError(t, err) + assert.NotEqual(t, "vault-ca-cert", config["VAULT_CACERT"]) + assert.NotEqual(t, "vault-client-cert", config["VAULT_CLIENT_CERT"]) + assert.NotEqual(t, "vault-client-key", config["VAULT_CLIENT_KEY"]) + assert.FileExists(t, config["VAULT_CACERT"]) + time.Sleep(time.Second * 6) + assert.NoFileExists(t, config["VAULT_CACERT"]) + assert.NoFileExists(t, config["VAULT_CLIENT_CERT"]) + assert.NoFileExists(t, config["VAULT_CLIENT_KEY"]) + close(removeCertFiles) + }) } func Test_buildKeyContext(t *testing.T) { diff --git a/tests/manifests/test-kms-vault-spec.yaml b/tests/manifests/test-kms-vault-spec.yaml index d9541f960533b..6848fe48d69be 100644 --- a/tests/manifests/test-kms-vault-spec.yaml +++ b/tests/manifests/test-kms-vault-spec.yaml @@ -7,4 +7,7 @@ spec: VAULT_BACKEND_PATH: rook/ver1 VAULT_SECRET_ENGINE: kv VAULT_SKIP_VERIFY: "true" + VAULT_CLIENT_KEY: "vault-client-key" + VAULT_CLIENT_CERT: "vault-client-cert" + VAULT_CACERT: "vault-ca-cert" tokenSecretName: rook-vault-token