diff --git a/Documentation/ceph-nfs-crd.md b/Documentation/ceph-nfs-crd.md index 9b10aa47fd129..488fe958c74f9 100644 --- a/Documentation/ceph-nfs-crd.md +++ b/Documentation/ceph-nfs-crd.md @@ -25,15 +25,6 @@ metadata: name: my-nfs namespace: rook-ceph spec: - rados: - # RADOS pool where NFS client recovery data and per-daemon configs are - # stored. In this example the data pool for the "myfs" filesystem is used. - # If using the object store example, the data pool would be - # "my-store.rgw.buckets.data". Note that this has nothing to do with where - # exported CephFS' or objectstores live. - pool: myfs-data0 - # RADOS namespace where NFS client recovery data is stored in the pool. - namespace: nfs-ns # Settings for the NFS server server: # the number of active NFS servers @@ -84,15 +75,6 @@ ceph dashboard set-ganesha-clusters-rados-pool-namespace [/:[/](,:[/])* ``` -## NFS Settings - -### RADOS Settings - -* `pool`: The pool where ganesha recovery backend and supplemental configuration objects will be stored -* `namespace`: The namespace in `pool` where ganesha recovery backend and supplemental configuration objects will be stored - -> **NOTE**: Don't use EC pools for NFS because ganesha uses omap in the recovery objects and grace db. EC pools do not support omap. - ## EXPORT Block Configuration All daemons within a cluster will share configuration with no exports defined, and that includes a RADOS object via: diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index bb478ba73104b..c96d847aa0251 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -5652,19 +5652,6 @@ spec: spec: description: NFSGaneshaSpec represents the spec of an nfs ganesha server properties: - rados: - description: RADOS is the Ganesha RADOS specification - properties: - namespace: - description: Namespace is the RADOS namespace where NFS client recovery data is stored. - type: string - pool: - description: Pool is the RADOS pool where NFS client recovery data is stored. - type: string - required: - - namespace - - pool - type: object server: description: Server is the Ganesha Server specification properties: @@ -6256,7 +6243,6 @@ spec: - active type: object required: - - rados - server type: object status: diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 00065717b2593..a024321917a32 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -5649,19 +5649,6 @@ spec: spec: description: NFSGaneshaSpec represents the spec of an nfs ganesha server properties: - rados: - description: RADOS is the Ganesha RADOS specification - properties: - namespace: - description: Namespace is the RADOS namespace where NFS client recovery data is stored. - type: string - pool: - description: Pool is the RADOS pool where NFS client recovery data is stored. - type: string - required: - - namespace - - pool - type: object server: description: Server is the Ganesha Server specification properties: @@ -6253,7 +6240,6 @@ spec: - active type: object required: - - rados - server type: object status: diff --git a/cluster/examples/kubernetes/ceph/nfs-test.yaml b/cluster/examples/kubernetes/ceph/nfs-test.yaml index 46770bdb62b62..7e89599c2d13f 100644 --- a/cluster/examples/kubernetes/ceph/nfs-test.yaml +++ b/cluster/examples/kubernetes/ceph/nfs-test.yaml @@ -4,13 +4,6 @@ metadata: name: my-nfs namespace: rook-ceph # namespace:cluster spec: - rados: - # RADOS pool where NFS client recovery data is stored. - # In this example the data pool for the "myfs" filesystem is used. - # If using the object store example, the data pool would be "my-store.rgw.buckets.data". - pool: myfs-data0 - # RADOS namespace where NFS client recovery data is stored in the pool. - namespace: nfs-ns # Settings for the NFS server server: # the number of active NFS servers diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 29141e4368e88..a2410da3ea591 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1564,22 +1564,11 @@ type CephNFSList struct { // NFSGaneshaSpec represents the spec of an nfs ganesha server type NFSGaneshaSpec struct { - // RADOS is the Ganesha RADOS specification - RADOS GaneshaRADOSSpec `json:"rados"` // Server is the Ganesha Server specification Server GaneshaServerSpec `json:"server"` } -// GaneshaRADOSSpec represents the specification of a Ganesha RADOS object -type GaneshaRADOSSpec struct { - // Pool is the RADOS pool where NFS client recovery data is stored. - Pool string `json:"pool"` - - // Namespace is the RADOS namespace where NFS client recovery data is stored. - Namespace string `json:"namespace"` -} - // GaneshaServerSpec represents the specification of a Ganesha Server type GaneshaServerSpec struct { // The number of active Ganesha servers diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index d52c33b31b0ce..3da51e38526ac 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -1747,22 +1747,6 @@ func (in *FilesystemsSpec) DeepCopy() *FilesystemsSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GaneshaRADOSSpec) DeepCopyInto(out *GaneshaRADOSSpec) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GaneshaRADOSSpec. -func (in *GaneshaRADOSSpec) DeepCopy() *GaneshaRADOSSpec { - if in == nil { - return nil - } - out := new(GaneshaRADOSSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GaneshaServerSpec) DeepCopyInto(out *GaneshaServerSpec) { *out = *in @@ -2170,7 +2154,6 @@ func (in *MonitoringSpec) DeepCopy() *MonitoringSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NFSGaneshaSpec) DeepCopyInto(out *NFSGaneshaSpec) { *out = *in - out.RADOS = in.RADOS in.Server.DeepCopyInto(&out.Server) return } diff --git a/pkg/operator/ceph/nfs/config.go b/pkg/operator/ceph/nfs/config.go index 67de962c66a7e..7bc9bec71830d 100644 --- a/pkg/operator/ceph/nfs/config.go +++ b/pkg/operator/ceph/nfs/config.go @@ -60,10 +60,10 @@ func getGaneshaConfigObject(n *cephv1.CephNFS, version cephver.CephVersion, name } func getRadosURL(n *cephv1.CephNFS, version cephver.CephVersion, name string) string { - url := fmt.Sprintf("rados://%s/", n.Spec.RADOS.Pool) + url := fmt.Sprintf("rados://%s/", radosPoolName) - if n.Spec.RADOS.Namespace != "" { - url += n.Spec.RADOS.Namespace + "/" + if n.Name != "" { + url += n.Name + "/" } url += getGaneshaConfigObject(n, version, name) @@ -71,9 +71,9 @@ func getRadosURL(n *cephv1.CephNFS, version cephver.CephVersion, name string) st } func (r *ReconcileCephNFS) generateKeyring(n *cephv1.CephNFS, name string) error { - osdCaps := fmt.Sprintf("allow rw pool=%s", n.Spec.RADOS.Pool) - if n.Spec.RADOS.Namespace != "" { - osdCaps = fmt.Sprintf("%s namespace=%s", osdCaps, n.Spec.RADOS.Namespace) + osdCaps := fmt.Sprintf("allow rw pool=%s", radosPoolName) + if n.Name != "" { + osdCaps = fmt.Sprintf("%s namespace=%s", osdCaps, n.Name) } caps := []string{"mon", "allow r", "osd", osdCaps} @@ -120,8 +120,8 @@ RADOS_KV { ceph_conf = '` + cephclient.DefaultConfigFilePath() + `'; userid = ` + userID + `; nodeid = ` + nodeID + `; - pool = "` + n.Spec.RADOS.Pool + `"; - namespace = "` + n.Spec.RADOS.Namespace + `"; + pool = "` + radosPoolName + `"; + namespace = "` + n.Name + `"; } RADOS_URLS { diff --git a/pkg/operator/ceph/nfs/controller_test.go b/pkg/operator/ceph/nfs/controller_test.go index c013e26d5d47b..ea7c210978ae9 100644 --- a/pkg/operator/ceph/nfs/controller_test.go +++ b/pkg/operator/ceph/nfs/controller_test.go @@ -90,10 +90,6 @@ func TestCephNFSController(t *testing.T) { Namespace: namespace, }, Spec: cephv1.NFSGaneshaSpec{ - RADOS: cephv1.GaneshaRADOSSpec{ - Pool: "foo", - Namespace: namespace, - }, Server: cephv1.GaneshaServerSpec{ Active: 1, }, diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index 11bfc8f2012e4..6c8e3556ef0a7 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -37,6 +37,7 @@ import ( const ( ganeshaRadosGraceCmd = "ganesha-rados-grace" + radosPoolName = ".nfs" ) var updateDeploymentAndWait = opmon.UpdateCephDeploymentAndWait @@ -130,8 +131,8 @@ func (r *ReconcileCephNFS) addRADOSConfigFile(n *cephv1.CephNFS, name string) er config := getGaneshaConfigObject(n, r.clusterInfo.CephVersion, name) cmd := "rados" args := []string{ - "--pool", n.Spec.RADOS.Pool, - "--namespace", n.Spec.RADOS.Namespace, + "--pool", radosPoolName, + "--namespace", n.Name, "--conf", cephclient.CephConfFilePath(r.context.ConfigDir, n.Namespace), } err := r.context.Executor.ExecuteCommand(cmd, append(args, "stat", config)...) @@ -165,7 +166,7 @@ func (r *ReconcileCephNFS) removeServerFromDatabase(nfs *cephv1.CephNFS, name st func (r *ReconcileCephNFS) runGaneshaRadosGrace(nfs *cephv1.CephNFS, name, action string) error { nodeID := getNFSNodeID(nfs, name) cmd := ganeshaRadosGraceCmd - args := []string{"--pool", nfs.Spec.RADOS.Pool, "--ns", nfs.Spec.RADOS.Namespace, action, nodeID} + args := []string{"--pool", radosPoolName, "--ns", nfs.Name, action, nodeID} env := []string{fmt.Sprintf("CEPH_CONF=%s", cephclient.CephConfFilePath(r.context.ConfigDir, nfs.Namespace))} return r.context.Executor.ExecuteCommandWithEnv(env, cmd, args...) @@ -254,6 +255,21 @@ func instanceName(n *cephv1.CephNFS, name string) string { return fmt.Sprintf("%s-%s-%s", AppName, n.Name, name) } +// create and enable .nfs RADOS pool +func createNFSRADOSPool(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) error { + args := []string{"osd", "pool", "create", ".nfs"} + _, err := cephclient.NewCephCommand(context, clusterInfo, args).Run() + if err != nil { + return err + } + args = []string{"osd", "pool", "application", "enable", ".nfs", "nfs"} + _, err = cephclient.NewCephCommand(context, clusterInfo, args).Run() + if err != nil { + return err + } + return nil +} + func validateGanesha(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, n *cephv1.CephNFS) error { // core properties if n.Name == "" { @@ -263,20 +279,19 @@ func validateGanesha(context *clusterd.Context, clusterInfo *cephclient.ClusterI return errors.New("missing namespace") } - // Client recovery properties - if n.Spec.RADOS.Pool == "" { - return errors.New("missing RADOS.pool") - } - // Ganesha server properties if n.Spec.Server.Active == 0 { return errors.New("at least one active server required") } - // The existence of the pool provided in n.Spec.RADOS.Pool is necessary otherwise addRADOSConfigFile() will fail - _, err := cephclient.GetPoolDetails(context, clusterInfo, n.Spec.RADOS.Pool) + // The existence of the .nfs pool is necessary otherwise addRADOSConfigFile() will fail + _, err := cephclient.GetPoolDetails(context, clusterInfo, radosPoolName) if err != nil { - return errors.Wrapf(err, "pool %q not found", n.Spec.RADOS.Pool) + // Create the .nfs pool if it doesn't already exist + err := createNFSRADOSPool(context, clusterInfo) + if err != nil { + return errors.Wrapf(err, ".nfs pool not found and unable to create it ") + } } return nil diff --git a/pkg/operator/ceph/nfs/spec_test.go b/pkg/operator/ceph/nfs/spec_test.go index cd171297eff97..f0bd4709f35a7 100644 --- a/pkg/operator/ceph/nfs/spec_test.go +++ b/pkg/operator/ceph/nfs/spec_test.go @@ -43,10 +43,6 @@ func TestDeploymentSpec(t *testing.T) { Namespace: "rook-ceph-test-ns", }, Spec: cephv1.NFSGaneshaSpec{ - RADOS: cephv1.GaneshaRADOSSpec{ - Pool: "myfs-data0", - Namespace: "nfs-test-ns", - }, Server: cephv1.GaneshaServerSpec{ Active: 3, Resources: v1.ResourceRequirements{ @@ -78,10 +74,6 @@ func TestDeploymentSpec(t *testing.T) { Namespace: namespace, }, Spec: cephv1.NFSGaneshaSpec{ - RADOS: cephv1.GaneshaRADOSSpec{ - Pool: "foo", - Namespace: namespace, - }, Server: cephv1.GaneshaServerSpec{ Active: 1, }, diff --git a/tests/framework/clients/nfs.go b/tests/framework/clients/nfs.go index c3b7ab2d84583..679f193f5141e 100644 --- a/tests/framework/clients/nfs.go +++ b/tests/framework/clients/nfs.go @@ -39,10 +39,10 @@ func CreateNFSOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests } // Create creates a filesystem in Rook -func (n *NFSOperation) Create(namespace, name, pool string, daemonCount int) error { +func (n *NFSOperation) Create(namespace, name string, daemonCount int) error { logger.Infof("creating the NFS daemons via CRD") - if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(name, pool, daemonCount)); err != nil { + if err := n.k8sh.ResourceOperation("apply", n.manifests.GetNFS(name, daemonCount)); err != nil { return err } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 2f73a0adf46d5..39d20dee94a66 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -40,7 +40,7 @@ type CephManifests interface { GetBlockSnapshotClass(snapshotClassName, reclaimPolicy string) string GetFileStorageSnapshotClass(snapshotClassName, reclaimPolicy string) string GetFilesystem(name string, activeCount int) string - GetNFS(name, pool string, daemonCount int) string + GetNFS(name string, daemonCount int) string GetRBDMirror(name string, daemonCount int) string GetObjectStore(name string, replicaCount, port int, tlsEnable bool) string GetObjectStoreUser(name, displayName, store string) string @@ -371,16 +371,13 @@ spec: } // GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config. -func (m *CephManifestsMaster) GetNFS(name, pool string, count int) string { +func (m *CephManifestsMaster) GetNFS(name string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephNFS metadata: name: ` + name + ` namespace: ` + m.settings.Namespace + ` spec: - rados: - pool: ` + pool + ` - namespace: nfs-ns server: active: ` + strconv.Itoa(count) } diff --git a/tests/framework/installer/ceph_manifests_v1.6.go b/tests/framework/installer/ceph_manifests_v1.6.go index 0bb4da38fca53..8188384733b8a 100644 --- a/tests/framework/installer/ceph_manifests_v1.6.go +++ b/tests/framework/installer/ceph_manifests_v1.6.go @@ -322,16 +322,13 @@ spec: } // GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config. -func (m *CephManifestsV1_6) GetNFS(name, pool string, count int) string { +func (m *CephManifestsV1_6) GetNFS(name string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephNFS metadata: name: ` + name + ` namespace: ` + m.settings.Namespace + ` spec: - rados: - pool: ` + pool + ` - namespace: nfs-ns server: active: ` + strconv.Itoa(count) } diff --git a/tests/integration/ceph_base_file_test.go b/tests/integration/ceph_base_file_test.go index f2acba483cb3a..e23933bd33382 100644 --- a/tests/integration/ceph_base_file_test.go +++ b/tests/integration/ceph_base_file_test.go @@ -305,7 +305,7 @@ func runFileE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.S func testNFSDaemons(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, settings *installer.TestCephSettings, filesystemName string) { name := "my-nfs" - err := helper.NFSClient.Create(settings.Namespace, name, filesystemName+"-data0", 2) + err := helper.NFSClient.Create(settings.Namespace, name, 2) require.Nil(s.T(), err) err = helper.NFSClient.Delete(settings.Namespace, name)