object: add support for user s3 key for cephobjectstoreuser

CephObjectStoreUser should optionally be able to reference
a secret where S3 key is defined. This enables us to
specify the accesskey and accesssecret rather than those
values being randomly generated.

Closes: rook#11563

Signed-off-by: parth-gr <partharora1010@gmail.com>

Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md

@@ -60,3 +60,76 @@ spec:
     * `user-policy`
     * `odic-provider`
     * `ratelimit`
+
+### CephObjectStoreUser Reference Secret
+
+A Kubernetes secret is created by Rook with the name, `rook-ceph-object-user-<store-name>-<user-name>` in the same namespace of cephobjectUser, where:
+
+* `store-name`: The object store name in which the user will be created. This matches the name of the objectstore CRD.
+* `user-name`: The metadata name of the cephObjectStoreUser
+
+This secret can be used to configure there aplications.
+Secret contains,
+`AccessKey`, `Credentials`, `Endpoint`, `Endpoint`.
+
+```console
+$ kubectl get secrets -nrook-ceph rook-ceph-object-user-my-store-my-user -o yaml
+apiVersion: v1
+data:
+  AccessKey: ***
+  Credentials: ***
+  Endpoint: ***
+  SecretKey: ***
+kind: Secret
+...
+...
+type: kubernetes.io/rook
+
+```
+User can use the above s3 keys to connect the rgw user to there application which require object storage.
+
+### User Specific CephObjectStoreUser Reference Secret
+
+If a specific user key and secret is desired instead of randomly generated credentials, a specific user key and secret can be specified for an object store user.
+
+Create or update the Kubernetes secret with name, `rook-ceph-object-user-<store-name>-<user-name>` in the same namespace of cephobjectUser, where:
+
+* `store-name`: The object store name in which the user will be created. This matches the name of the objectstore CRD.
+* `user-name`: The metadata name of the cephObjectStoreUser
+
+#### Secret details:
+
+i) The annotations as `rook.io/source-of-truth: secret` specified and type as `type: "kubernetes.io/rook"`.
+
+ii) (optional) The array of `SecretKeys` which contains all the truted access and secret keys. If any key is present in the ceph user and not present in the `SecretKeys`, it will be removed from the ceph user too.
+
+If `SecretKeys` is left empty, it will be auto updated by the latest AccessKey and SecretKey and will remove all the other keys from the ceph user.
+
+```console
+`SecretKeys`
+[{"access_key":"IE58RNT71Y2F1EQE80RA","secret_key":"cULyMz5dCpX18dPsJhpIKay7vcDNRNJWJPu8VqUA"}, {"access_key":"IE58RNT71Y2F1EQE80RC","secret_key":"cULyMz5dCpX18dPsJhpIKay7vcDNRNJWJPu8VqUA"}]
+
+!!! note
+    All the data values should be converted to base64 format and then updated.
+
+Example Secret:
+```console
+kubectl create -f
+apiVersion: v1
+kind: Secret
+metadata:
+  name: rook-ceph-object-user-my-store-my-user
+  namespace: rook-ceph
+  annotations:
+    rook.io/source-of-truth: secret
+data:
+  AccessKey: ***
+  SecretKey: ***
+  Endpoint: ***
+  Credentials: ***
+type: "kubernetes.io/rook"
+```
+
+!!! note
+    Be careful when updating Kubernetes secret values. Mistakes entering info can cause errors reconciling the CephObjectStore.
+    And ignore the endpoint when `rook.io/source-of-truth: secret` is set, and instead use the `CephObjectStore.Status.Endpoints` list to retrieve endpoints for applications.

go.mod

@@ -67,7 +67,7 @@ require (
 )
 
 require (
-	emperror.dev/errors v0.8.1 // indirect
+	emperror.dev/errors v0.8.1
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/ansel1/merry v1.8.0 // indirect
 	github.com/ansel1/merry/v2 v2.2.0 // indirect

pkg/operator/ceph/object/secret/secret.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2016 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package secret
+
+import (
+	"encoding/json"
+	"strings"
+
+	"emperror.dev/errors"
+	"github.com/ceph/go-ceph/rgw/admin"
+)
+
+const (
+	//#nosec G101 -- This is the name of an annotation key, not a secret
+	updateObjectUserSecretAnnotation = "rook.io/source-of-truth"
+	sourceOfTruthSecret              = "secret"
+)
+
+func ValidateCredentials(accessKey, secretKey string, credentials []admin.UserKeySpec) ([]admin.UserKeySpec, error) {
+	// if len is 0 just update the credentials array with the accessKey and secretKey key
+	if len(credentials) == 0 {
+		credentials = []admin.UserKeySpec{
+			{AccessKey: accessKey,
+				SecretKey: secretKey}}
+		return credentials, nil
+	}
+	// match if the credentials array contains the accessKey and secretKey key
+	for i := range credentials {
+		if credentials[i].AccessKey == accessKey && credentials[0].SecretKey == secretKey {
+			return credentials, nil
+		}
+	}
+
+	return credentials, errors.Errorf("secret keys data is invalid, please update the secret with cvalid format")
+}
+
+func UnmarshalKeys(credentials []byte) ([]admin.UserKeySpec, error) {
+	var userKeys []admin.UserKeySpec
+	err := json.Unmarshal(credentials, &userKeys)
+	if err != nil {
+		return userKeys, errors.Wrapf(err, "unable to unmarshal credentials from the object secret")
+	}
+	return userKeys, nil
+}
+
+func ForceUpdateObjectUserSecret(annotations map[string]string) bool {
+	if value, found := annotations[updateObjectUserSecretAnnotation]; found {
+		if strings.EqualFold(value, sourceOfTruthSecret) {
+			return true
+		}
+	}
+	return false
+}
+
+func UpdatecredentialsUserId(uid, displayName string, credentials []admin.UserKeySpec) []admin.UserKeySpec {
+	updatedCredentials := credentials
+	for i := range credentials {
+		updatedCredentials[i].UID = uid
+		updatedCredentials[i].User = displayName
+	}
+
+	return updatedCredentials
+}

pkg/operator/ceph/object/user.go

@@ -241,17 +241,23 @@ func GenerateCephUserSecretName(store, username string) string {
 	return fmt.Sprintf("rook-ceph-object-user-%s-%s", store, username)
 }
 
-func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) *corev1.Secret {
+func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) (*corev1.Secret, error) {
 	secretName := GenerateCephUserSecretName(storeName, userConfig.ID)
+	secretKeys, err := json.Marshal(userConfig.Keys)
+	if err != nil {
+		return &corev1.Secret{}, err
+	}
 	// Store the keys in a secret
 	secrets := map[string]string{
-		"AccessKey": userConfig.Keys[0].AccessKey,
-		"SecretKey": userConfig.Keys[0].SecretKey,
-		"Endpoint":  endpoint,
+		"AccessKey":  userConfig.Keys[0].AccessKey,
+		"SecretKey":  userConfig.Keys[0].SecretKey,
+		"SecretKeys": string(secretKeys),
+		"Endpoint":   endpoint,
 	}
 	if tlsSecretName != "" {
 		secrets["SSLCertSecretName"] = tlsSecretName
 	}
+
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      secretName,
@@ -266,15 +272,18 @@ func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeNa
 		StringData: secrets,
 		Type:       k8sutil.RookType,
 	}
-	return secret
+
+	return secret, nil
 }
 
 func ReconcileCephUserSecret(ctx context.Context, k8sclient client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) (reconcile.Result, error) {
 	// Generate Kubernetes Secret
-	secret := generateCephUserSecret(userConfig, endpoint, namespace, storeName, tlsSecretName)
-
+	secret, err := generateCephUserSecret(userConfig, endpoint, namespace, storeName, tlsSecretName)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to get the ceph object user secret %q", GenerateCephUserSecretName(storeName, userConfig.ID))
+	}
 	// Set owner ref to the object store user object
-	err := controllerutil.SetControllerReference(ownerRef, secret, scheme)
+	err = controllerutil.SetControllerReference(ownerRef, secret, scheme)
 	if err != nil {
 		return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of ceph object user secret %q", secret.Name)
 	}