object: add support for user s3 key for cephobjectstoreuser

CephObjectStoreUser should optionally be able to reference
a secret where S3 key is defined. This enables us to
specify the accesskey and accesssecret rather than those
values being randomly generated.

Closes: rook#11563

Signed-off-by: parth-gr <partharora1010@gmail.com>

Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md

@@ -60,3 +60,78 @@ spec:
     * `user-policy`
     * `odic-provider`
     * `ratelimit`
+
+### CephObjectStoreUser Reference Secret
+
+A Kubernetes secret is created by Rook with the name, `rook-ceph-object-user-<store-name>-<user-name>` in the same namespace of cephobjectUser, where:
+
+* `store-name`: The object store name in which the user will be created. This matches the name of the objectstore CRD.
+* `user-name`: The metadata name of the cephObjectStoreUser
+
+Application pods can use the secret to configure access to the object store.
+
+```yaml
+apiVersion: v1
+data:
+  AccessKey: ***       # [1]
+  Credentials: ***     # [2]
+  Endpoint: ***        # [3]
+  SecretKey: ***       # [4]
+kind: Secret
+...
+...
+type: kubernetes.io/rook
+
+```
+
+1. `AccessKey`: User access key for ceph S3.
+2. `SecretKey`: User secret key for ceph S3.
+3. `Credentials`: Comma separated values of all access and secret keys.
+4. `Endpoint`: Endpoint for ceph S3.
+
+#### User-Specified Reference Secret
+
+AccessKey, SecretKey, and Credentials can be specified before CephObjectStore creation or modified afterwards to support user-specified keys and key rotation. Endpoint cannot be overridden by users and will always be updated by Rook to match the latest CephObjectStore information.
+
+Create or update the Kubernetes secret with name, `rook-ceph-object-user-<store-name>-<user-name>` in the same namespace of cephobjectUser,
+
+Steps:
+
+i) Specify the annotations as `rook.io/source-of-truth: secret` and type as `type: "kubernetes.io/rook"`.
+
+ii) Specify the user defined `AccessKey` and `SecretKey`.
+
+iii) (optional) The array of `Credentials` which contains all the trusted access and secret keys. Make sure to also add the user defined `AccessKey` and `SecretKey` of step ii).
+
+`Credentials`
+```console
+[{"access_key":"IE58RNT71Y2F1EQE80RA","secret_key":"cULyMz5dCpX18dPsJhpIKay7vcDNRNJWJPu8VqUA"}, {"access_key":"IE58RNT71Y2F1EQE80RC","secret_key":"cULyMz5dCpX18dPsJhpIKay7vcDNRNJWJPu8VqUA"}]
+```
+
+If any key is present in the ceph user and not present in the `Credentials`, it will be removed from the ceph user too.
+If `Credentials` is left empty, it will be auto updated by the latest AccessKey and SecretKey and will remove all the other keys from the ceph user.
+
+!!! note
+    Secret data usually needs to be converted to base64 format.
+
+Example Secret:
+
+```console
+kubectl create -f
+apiVersion: v1
+kind: Secret
+metadata:
+  name: rook-ceph-object-user-my-store-my-user
+  namespace: rook-ceph
+  annotations:
+    rook.io/source-of-truth: secret
+data:
+  AccessKey: ***
+  SecretKey: ***
+  Endpoint: ***
+  Credentials: ***
+type: "kubernetes.io/rook"
+```
+
+!!! note
+    Be careful when updating Kubernetes secret values. Mistakes entering info can cause errors reconciling the CephObjectStore.

go.mod

@@ -67,7 +67,7 @@ require (
 )
 
 require (
-	emperror.dev/errors v0.8.1 // indirect
+	emperror.dev/errors v0.8.1
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/ansel1/merry v1.8.0 // indirect
 	github.com/ansel1/merry/v2 v2.2.0 // indirect

pkg/operator/ceph/object/secret/secret.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2016 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package secret
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/ceph/go-ceph/rgw/admin"
+	"github.com/pkg/errors"
+)
+
+const (
+	//#nosec G101 -- This is the name of an annotation key, not a secret
+	UpdateObjectUserSecretAnnotation = "rook.io/source-of-truth"
+	SourceOfTruthSecret              = "secret"
+)
+
+// Credential is the user credential values
+type Credential struct {
+	AccessKey string `json:"access_key" url:"access-key"`
+	SecretKey string `json:"secret_key" url:"secret-key"`
+}
+
+func ValidateCredentials(accessKey, secretKey string, credentials []Credential) ([]Credential, error) {
+	// if len is 0 just update the credentials array with the accessKey and secretKey key
+	if len(credentials) == 0 {
+		credentials = []Credential{
+			{AccessKey: accessKey,
+				SecretKey: secretKey}}
+		return credentials, nil
+	}
+	// match if the credentials array contains the accessKey and secretKey key
+	for i := range credentials {
+		if credentials[i].AccessKey == accessKey && credentials[0].SecretKey == secretKey {
+			return credentials, nil
+		}
+	}
+
+	return credentials, errors.Errorf("secret keys data is invalid, please update the secret with cvalid format")
+}
+
+func UnmarshalKeys(credentials []byte) ([]Credential, error) {
+	var userKeys []Credential
+	err := json.Unmarshal(credentials, &userKeys)
+	if err != nil {
+		return userKeys, errors.Wrapf(err, "unable to unmarshal credentials from the object secret")
+	}
+	return userKeys, nil
+}
+
+func GetUserDefinedSecretAnnotations(annotations map[string]string) bool {
+	if value, found := annotations[UpdateObjectUserSecretAnnotation]; found {
+		if strings.EqualFold(value, SourceOfTruthSecret) {
+			return true
+		}
+	}
+	return false
+}
+
+func UpdatecredentialsUserId(uid, displayName string, credentials []Credential) []admin.UserKeySpec {
+	var updatedCredentials []admin.UserKeySpec
+	for i := range credentials {
+		userSpec := admin.UserKeySpec{User: displayName, UID: uid, AccessKey: credentials[i].AccessKey, SecretKey: credentials[i].SecretKey}
+		updatedCredentials = append(updatedCredentials, userSpec)
+	}
+
+	return updatedCredentials
+}

pkg/operator/ceph/object/user.go

@@ -241,17 +241,23 @@ func GenerateCephUserSecretName(store, username string) string {
 	return fmt.Sprintf("rook-ceph-object-user-%s-%s", store, username)
 }
 
-func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) *corev1.Secret {
+func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) (*corev1.Secret, error) {
 	secretName := GenerateCephUserSecretName(storeName, userConfig.ID)
+	secretKeys, err := json.Marshal(userConfig.Keys)
+	if err != nil {
+		return &corev1.Secret{}, err
+	}
 	// Store the keys in a secret
 	secrets := map[string]string{
-		"AccessKey": userConfig.Keys[0].AccessKey,
-		"SecretKey": userConfig.Keys[0].SecretKey,
-		"Endpoint":  endpoint,
+		"AccessKey":  userConfig.Keys[0].AccessKey,
+		"SecretKey":  userConfig.Keys[0].SecretKey,
+		"SecretKeys": string(secretKeys),
+		"Endpoint":   endpoint,
 	}
 	if tlsSecretName != "" {
 		secrets["SSLCertSecretName"] = tlsSecretName
 	}
+
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      secretName,
@@ -266,15 +272,18 @@ func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeNa
 		StringData: secrets,
 		Type:       k8sutil.RookType,
 	}
-	return secret
+
+	return secret, nil
 }
 
 func ReconcileCephUserSecret(ctx context.Context, k8sclient client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) (reconcile.Result, error) {
 	// Generate Kubernetes Secret
-	secret := generateCephUserSecret(userConfig, endpoint, namespace, storeName, tlsSecretName)
-
+	secret, err := generateCephUserSecret(userConfig, endpoint, namespace, storeName, tlsSecretName)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to get the ceph object user secret %q", GenerateCephUserSecretName(storeName, userConfig.ID))
+	}
 	// Set owner ref to the object store user object
-	err := controllerutil.SetControllerReference(ownerRef, secret, scheme)
+	err = controllerutil.SetControllerReference(ownerRef, secret, scheme)
 	if err != nil {
 		return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of ceph object user secret %q", secret.Name)
 	}