ceph: update CephNFS to use ".nfs" pool in newer ceph versions

This commit updates the CephNFS CR to make the RADOS settings optional
for Ceph versions above 16.2.6 due to the NFS module changes in Ceph.
The changes in Ceph make it so the RADOS pool is always ".nfs" and the
RADOS namespace is always the name of the NFS cluster.

Closes: #8450
Signed-off-by: Joseph Sawaya <jsawaya@redhat.com>

Documentation/ceph-nfs-crd.md

@@ -25,6 +25,8 @@ metadata:
   name: my-nfs
   namespace: rook-ceph
 spec:
+  # rados property is not used in versions of Ceph equal to or greater than 
+  # 16.2.6, see note in RADOS settings section below.
   rados:
     # RADOS pool where NFS client recovery data and per-daemon configs are
     # stored. In this example the data pool for the "myfs" filesystem is used.
@@ -91,6 +93,8 @@ ceph dashboard set-ganesha-clusters-rados-pool-namespace <cluster_id>:<pool_name
 * `pool`: The pool where ganesha recovery backend and supplemental configuration objects will be stored
 * `namespace`: The namespace in `pool` where ganesha recovery backend and supplemental configuration objects will be stored
 
+> **NOTE**: The RADOS settings aren't used in Ceph versions equal to or greater than Pacific 16.2.6, default values are used instead ".nfs" for the RADOS pool and the CephNFS CR's name for the RADOS namespace. However, RADOS settings are mandatory for versions preceding Pacific 16.2.6.
+
 > **NOTE**: Don't use EC pools for NFS because ganesha uses omap in the recovery objects and grace db. EC pools do not support omap.
 
 ## EXPORT Block Configuration

cluster/charts/rook-ceph/templates/resources.yaml

@@ -6256,7 +6256,6 @@ spec:
                     - active
                   type: object
               required:
-                - rados
                 - server
               type: object
             status:

cluster/examples/kubernetes/ceph/crds.yaml

@@ -6253,7 +6253,6 @@ spec:
                     - active
                   type: object
               required:
-                - rados
                 - server
               type: object
             status:

cluster/examples/kubernetes/ceph/nfs-test.yaml

@@ -4,6 +4,7 @@ metadata:
   name: my-nfs
   namespace: rook-ceph # namespace:cluster
 spec:
+  # rados settings aren't necessary in Ceph Versions equal to or greater than Pacific 16.2.6
   rados:
     # RADOS pool where NFS client recovery data is stored.
     # In this example the data pool for the "myfs" filesystem is used.

pkg/apis/ceph.rook.io/v1/types.go

@@ -1565,7 +1565,7 @@ type CephNFSList struct {
 // NFSGaneshaSpec represents the spec of an nfs ganesha server
 type NFSGaneshaSpec struct {
 	// RADOS is the Ganesha RADOS specification
-	RADOS GaneshaRADOSSpec `json:"rados"`
+	RADOS GaneshaRADOSSpec `json:"rados,omitempty"`
 
 	// Server is the Ganesha Server specification
 	Server GaneshaServerSpec `json:"server"`

pkg/operator/ceph/nfs/controller.go

@@ -31,6 +31,7 @@ import (
 	opconfig "github.com/rook/rook/pkg/operator/ceph/config"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/ceph/reporting"
+	"github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -204,6 +205,11 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul
 	}
 	r.clusterInfo.CephVersion = currentCephVersion
 
+	if r.clusterInfo.CephVersion.IsAtLeast(version.CephVersion{Major: 16, Minor: 2, Extra: 6}) {
+		cephNFS.Spec.RADOS.Pool = defaultRadosPoolName
+		cephNFS.Spec.RADOS.Namespace = cephNFS.Name
+	}
+
 	// DELETE: the CR was deleted
 	if !cephNFS.GetDeletionTimestamp().IsZero() {
 		logger.Infof("deleting ceph nfs %q", cephNFS.Name)

pkg/operator/ceph/nfs/nfs.go

@@ -20,6 +20,7 @@ package nfs
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/banzaicloud/k8s-objectmatcher/patch"
 	"github.com/pkg/errors"
@@ -28,6 +29,7 @@ import (
 	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
 	opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	"github.com/rook/rook/pkg/operator/ceph/config"
+	"github.com/rook/rook/pkg/operator/ceph/version"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	v1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -37,6 +39,7 @@ import (
 
 const (
 	ganeshaRadosGraceCmd = "ganesha-rados-grace"
+	defaultRadosPoolName = ".nfs"
 )
 
 var updateDeploymentAndWait = opmon.UpdateCephDeploymentAndWait
@@ -254,6 +257,21 @@ func instanceName(n *cephv1.CephNFS, name string) string {
 	return fmt.Sprintf("%s-%s-%s", AppName, n.Name, name)
 }
 
+// create and enable .nfs RADOS pool
+func createDefaultNFSRADOSPool(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error {
+	args := []string{"osd", "pool", "create", defaultRadosPoolName}
+	_, err := cephclient.NewCephCommand(context, clusterInfo, args).Run()
+	if err != nil {
+		return err
+	}
+	args = []string{"osd", "pool", "application", "enable", defaultRadosPoolName, "nfs"}
+	_, err = cephclient.NewCephCommand(context, clusterInfo, args).Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func validateGanesha(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, n *cephv1.CephNFS) error {
 	// core properties
 	if n.Name == "" {
@@ -268,6 +286,10 @@ func validateGanesha(context *clusterd.Context, clusterInfo *cephclient.ClusterI
 		return errors.New("missing RADOS.pool")
 	}
 
+	if n.Spec.RADOS.Namespace == "" {
+		return errors.New("missing RADOS.namespace")
+	}
+
 	// Ganesha server properties
 	if n.Spec.Server.Active == 0 {
 		return errors.New("at least one active server required")
@@ -276,6 +298,14 @@ func validateGanesha(context *clusterd.Context, clusterInfo *cephclient.ClusterI
 	// The existence of the pool provided in n.Spec.RADOS.Pool is necessary otherwise addRADOSConfigFile() will fail
 	_, err := cephclient.GetPoolDetails(context, clusterInfo, n.Spec.RADOS.Pool)
 	if err != nil {
+		if strings.Contains(err.Error(), "unrecognized pool") {
+			if clusterInfo.CephVersion.IsAtLeast(version.CephVersion{Major: 16, Minor: 2, Extra: 6}) {
+				err := createDefaultNFSRADOSPool(context, clusterInfo)
+				if err != nil {
+					return errors.Wrapf(err, "failed to find %q pool and unable to create it", n.Spec.RADOS.Pool)
+				}
+			}
+		}
 		return errors.Wrapf(err, "pool %q not found", n.Spec.RADOS.Pool)
 	}