object: add support for user s3 key for cephobjectstoreuser

CephObjectStoreUser should optionally be able to reference
a secret where S3 key is defined. This enables us to
specify the accesskey and accesssecret rather than those
values being randomly generated.

Closes: rook#11563

Signed-off-by: parth-gr <partharora1010@gmail.com>

Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md

@@ -60,3 +60,28 @@ spec:
     * `user-policy`
     * `odic-provider`
     * `ratelimit`
+
+### CephObjectStoreUser Reference Secret
+
+If a specific user key and secret is desired instead of randomly generated credentials, a specific user key and secret can be specified for an object store user.
+
+Create or update the Kubernetes secret with name, `rook-ceph-object-user-<store-name>-<user-name>` in the same namespace of cephobjectUser, where:
+
+* `store-name`: The object store name in which the user will be created. This matches the name of the objectstore CRD.
+* `user-name`: The metadata name of the cephObjectStoreUser
+
+```console
+kubectl create -f
+apiVersion: v1
+kind: Secret
+metadata:
+  name: rook-ceph-object-user-my-store-my-user
+  namespace: rook-ceph
+  annotations:
+    administration.rook.io/source-of-truth: secret
+data:
+  AccessKey: ***
+  SecretKey: ***
+  Endpoint: ***
+type: "kubernetes.io/rook"
+```

pkg/operator/ceph/object/user.go

@@ -45,6 +45,8 @@ const (
 	ErrorCodeFileExists = 17
 )
 
+var UserKeysSource = false
+
 // An ObjectUser defines the details of an object store user.
 type ObjectUser struct {
 	UserID       string              `json:"userId"`
@@ -252,6 +254,7 @@ func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeNa
 	if tlsSecretName != "" {
 		secrets["SSLCertSecretName"] = tlsSecretName
 	}
+
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      secretName,
@@ -266,6 +269,10 @@ func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeNa
 		StringData: secrets,
 		Type:       k8sutil.RookType,
 	}
+	if UserKeysSource {
+		secret.Annotations = map[string]string{"administration.rook.io/source-of-truth": "secret"}
+	}
+
 	return secret
 }
 

pkg/operator/ceph/object/user/controller.go

@@ -292,10 +292,35 @@ func (r *ReconcileObjectStoreUser) createOrUpdateCephUser(u *cephv1.CephObjectSt
 
 	logCreateOrUpdate := fmt.Sprintf("retrieved existing ceph object user %q", u.Name)
 	var user admin.User
+	var userKeys []admin.UserKeySpec
 	var err error
+
+	// get the user defined k8s s3 keys if exists
+	secretName := object.GenerateCephUserSecretName(u.Spec.Store, u.Name)
+	namspacedName := types.NamespacedName{Namespace: u.Namespace, Name: secretName}
+	cephObjectStoreUserSecret := &corev1.Secret{}
+	err = r.client.Get(r.clusterInfo.Context, namspacedName, cephObjectStoreUserSecret)
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			logger.Debugf("no user secret %q provided for cephobjectuser", secretName)
+		} else {
+			return errors.Wrapf(err, "failed to get user cephobjectuser secret %q", secretName)
+		}
+	} else {
+		userKeys = []admin.UserKeySpec{
+			{AccessKey: string(cephObjectStoreUserSecret.Data["AccessKey"]),
+				SecretKey: string(cephObjectStoreUserSecret.Data["SecretKey"])},
+		}
+		if cephObjectStoreUserSecret.Annotations["rook.io/source-of-truth"] == "secret" {
+			object.UserKeysSource = true
+		}
+	}
+
 	user, err = r.objContext.AdminOpsClient.GetUser(r.opManagerContext, *r.userConfig)
 	if err != nil {
 		if errors.Is(err, admin.ErrNoSuchUser) {
+			// if secret exists use the user specified keys
+			r.userConfig.Keys = userKeys
 			user, err = r.objContext.AdminOpsClient.CreateUser(r.opManagerContext, *r.userConfig)
 			if err != nil {
 				return errors.Wrapf(err, "failed to create ceph object user %v", &r.userConfig.ID)
@@ -306,6 +331,11 @@ func (r *ReconcileObjectStoreUser) createOrUpdateCephUser(u *cephv1.CephObjectSt
 		}
 	}
 
+	if object.UserKeysSource {
+		// if secret exists and source of truth is secret then use the user specified keys
+		r.userConfig.Keys = userKeys
+	}
+
 	// Update max bucket if necessary
 	logger.Tracef("user capabilities(id: %s, caps: %#v, user caps: %s, op mask: %s)",
 		user.ID, user.Caps, user.UserCaps, user.OpMask)
@@ -363,11 +393,11 @@ func (r *ReconcileObjectStoreUser) createOrUpdateCephUser(u *cephv1.CephObjectSt
 	}
 
 	// Set access and secret key
-	if r.userConfig.Keys == nil {
+	if r.userConfig.Keys == nil || !object.UserKeysSource {
 		r.userConfig.Keys = make([]admin.UserKeySpec, 1)
+		r.userConfig.Keys[0].AccessKey = user.Keys[0].AccessKey
+		r.userConfig.Keys[0].SecretKey = user.Keys[0].SecretKey
 	}
-	r.userConfig.Keys[0].AccessKey = user.Keys[0].AccessKey
-	r.userConfig.Keys[0].SecretKey = user.Keys[0].SecretKey
 	logger.Info(logCreateOrUpdate)
 
 	return nil

pkg/operator/ceph/object/user/controller_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/coreos/pkg/capnslog"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
+	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/operator/test"
 
@@ -491,15 +492,22 @@ func TestCreateOrUpdateCephUser(t *testing.T) {
 			return nil, fmt.Errorf("unexpected request: %q. method %q. path %q", req.URL.RawQuery, req.Method, req.URL.Path)
 		},
 	}
+	s := scheme.Scheme
+	cl := fake.NewClientBuilder().WithScheme(s).Build()
 	adminClient, err := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient)
 	assert.NoError(t, err)
 	userConfig := generateUserConfig(objectUser)
+	clusterInfo := &cephclient.ClusterInfo{
+		Context: context.TODO(),
+	}
 	r := &ReconcileObjectStoreUser{
 		objContext: &cephobject.AdminOpsContext{
 			AdminOpsClient: adminClient,
 		},
 		userConfig:       &userConfig,
 		opManagerContext: context.TODO(),
+		client:           cl,
+		clusterInfo:      clusterInfo,
 	}
 	maxsize, err := resource.ParseQuantity(maxsizestr)
 	assert.NoError(t, err)