diff --git a/Documentation/ceph-nfs-crd.md b/Documentation/ceph-nfs-crd.md index 9b10aa47fd129..a2613e5130a8a 100644 --- a/Documentation/ceph-nfs-crd.md +++ b/Documentation/ceph-nfs-crd.md @@ -25,6 +25,8 @@ metadata: name: my-nfs namespace: rook-ceph spec: + # rados property is not used in versions of Ceph equal to or greater than + # 16.2.6, see note in RADOS settings section below. rados: # RADOS pool where NFS client recovery data and per-daemon configs are # stored. In this example the data pool for the "myfs" filesystem is used. @@ -91,6 +93,8 @@ ceph dashboard set-ganesha-clusters-rados-pool-namespace : **NOTE**: The RADOS settings aren't used in Ceph versions equal to or greater than Pacific 16.2.6, default values are used instead ".nfs" for the RADOS pool and the CephNFS CR's name for the RADOS namespace. However, RADOS settings are mandatory for versions preceding Pacific 16.2.6. + > **NOTE**: Don't use EC pools for NFS because ganesha uses omap in the recovery objects and grace db. EC pools do not support omap. ## EXPORT Block Configuration diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index bb478ba73104b..5cc1de38a14ca 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -6256,7 +6256,6 @@ spec: - active type: object required: - - rados - server type: object status: diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 00065717b2593..9f31000f1574e 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -6253,7 +6253,6 @@ spec: - active type: object required: - - rados - server type: object status: diff --git a/cluster/examples/kubernetes/ceph/nfs-test.yaml b/cluster/examples/kubernetes/ceph/nfs-test.yaml index 46770bdb62b62..02dbc6fb6013e 100644 --- a/cluster/examples/kubernetes/ceph/nfs-test.yaml +++ b/cluster/examples/kubernetes/ceph/nfs-test.yaml @@ -4,6 +4,7 @@ metadata: name: my-nfs namespace: rook-ceph # namespace:cluster spec: + # rados settings aren't necessary in Ceph Versions equal to or greater than Pacific 16.2.6 rados: # RADOS pool where NFS client recovery data is stored. # In this example the data pool for the "myfs" filesystem is used. diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 29141e4368e88..64abb6ff445c0 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1565,7 +1565,7 @@ type CephNFSList struct { // NFSGaneshaSpec represents the spec of an nfs ganesha server type NFSGaneshaSpec struct { // RADOS is the Ganesha RADOS specification - RADOS GaneshaRADOSSpec `json:"rados"` + RADOS GaneshaRADOSSpec `json:"rados,omitempty"` // Server is the Ganesha Server specification Server GaneshaServerSpec `json:"server"` diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index 32f5cd913834b..01987c0d70d34 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -31,6 +31,7 @@ import ( opconfig "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/reporting" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -204,6 +205,11 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul } r.clusterInfo.CephVersion = currentCephVersion + if r.clusterInfo.CephVersion.IsAtLeast(version.CephVersion{Major: 16, Minor: 2, Extra: 6}) { + cephNFS.Spec.RADOS.Pool = defaultRadosPoolName + cephNFS.Spec.RADOS.Namespace = cephNFS.Name + } + // DELETE: the CR was deleted if !cephNFS.GetDeletionTimestamp().IsZero() { logger.Infof("deleting ceph nfs %q", cephNFS.Name) diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index 11bfc8f2012e4..20a4face85c91 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -20,6 +20,7 @@ package nfs import ( "context" "fmt" + "strings" "github.com/banzaicloud/k8s-objectmatcher/patch" "github.com/pkg/errors" @@ -28,6 +29,7 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -37,6 +39,7 @@ import ( const ( ganeshaRadosGraceCmd = "ganesha-rados-grace" + defaultRadosPoolName = ".nfs" ) var updateDeploymentAndWait = opmon.UpdateCephDeploymentAndWait @@ -254,6 +257,21 @@ func instanceName(n *cephv1.CephNFS, name string) string { return fmt.Sprintf("%s-%s-%s", AppName, n.Name, name) } +// create and enable .nfs RADOS pool +func createDefaultNFSRADOSPool(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error { + args := []string{"osd", "pool", "create", defaultRadosPoolName} + _, err := cephclient.NewCephCommand(context, clusterInfo, args).Run() + if err != nil { + return err + } + args = []string{"osd", "pool", "application", "enable", defaultRadosPoolName, "nfs"} + _, err = cephclient.NewCephCommand(context, clusterInfo, args).Run() + if err != nil { + return err + } + return nil +} + func validateGanesha(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, n *cephv1.CephNFS) error { // core properties if n.Name == "" { @@ -268,6 +286,10 @@ func validateGanesha(context *clusterd.Context, clusterInfo *cephclient.ClusterI return errors.New("missing RADOS.pool") } + if n.Spec.RADOS.Namespace == "" { + return errors.New("missing RADOS.namespace") + } + // Ganesha server properties if n.Spec.Server.Active == 0 { return errors.New("at least one active server required") @@ -276,6 +298,14 @@ func validateGanesha(context *clusterd.Context, clusterInfo *cephclient.ClusterI // The existence of the pool provided in n.Spec.RADOS.Pool is necessary otherwise addRADOSConfigFile() will fail _, err := cephclient.GetPoolDetails(context, clusterInfo, n.Spec.RADOS.Pool) if err != nil { + if strings.HasPrefix(err.Error(), "failed to get pool") { + if clusterInfo.CephVersion.IsAtLeast(version.CephVersion{Major: 16, Minor: 2, Extra: 6}) { + err := createDefaultNFSRADOSPool(context, clusterInfo) + if err != nil { + return errors.Wrapf(err, "failed to find %q pool and unable to create it", n.Spec.RADOS.Pool) + } + } + } return errors.Wrapf(err, "pool %q not found", n.Spec.RADOS.Pool) }