mgr: run the watch side car config on /etc/ceph

We don't need to use /var/lib/rook for the ceph config and keyring, so
let's use /etc/ceph which has the right permissions too. Also, we need
to run with the ceph user since the rook image defaults to the "rook"
user.

Signed-off-by: Sébastien Han <seb@redhat.com>

cmd/rook/ceph/mgr.go

@@ -68,6 +68,7 @@ func runMgrSidecar(cmd *cobra.Command, args []string) error {
 	rook.SetLogLevel()
 
 	context := createContext()
+	context.ConfigDir = client.DefaultConfigDir
 	clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
 	rook.LogStartupInfo(mgrSidecarCmd.Flags())
 

pkg/daemon/ceph/client/config.go

@@ -91,6 +91,9 @@ func GenerateConnectionConfig(context *clusterd.Context, cluster *ClusterInfo) (
 // some subset of settings.
 func GenerateConnectionConfigWithSettings(context *clusterd.Context, clusterInfo *ClusterInfo, settings *CephConfig) (string, error) {
 	root := path.Join(context.ConfigDir, clusterInfo.Namespace)
+	if context.ConfigDir == DefaultConfigDir {
+		root = context.ConfigDir
+	}
 	keyringPath := path.Join(root, fmt.Sprintf("%s.keyring", clusterInfo.CephCred.Username))
 	err := writeKeyring(CephKeyring(clusterInfo.CephCred), keyringPath)
 	if err != nil {

pkg/operator/ceph/cluster/mgr/spec.go

@@ -227,12 +227,16 @@ func (c *Cluster) makeMgrSidecarContainer(mgrConfig *mgrConfig) v1.Container {
 		{Name: "ROOK_CEPH_VERSION", Value: "ceph version " + c.clusterInfo.CephVersion.String()},
 	}
 
+	runAsCeph := int64(167)
 	return v1.Container{
 		Args:      []string{"ceph", "mgr", "watch-active"},
 		Name:      "watch-active",
 		Image:     c.rookVersion,
 		Env:       envVars,
 		Resources: cephv1.GetMgrSidecarResources(c.spec.Resources),
+		SecurityContext: &v1.SecurityContext{
+			RunAsUser: &runAsCeph,
+		},
 	}
 }