diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 5923e746c23df..f92220e6c9bac 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -4973,6 +4973,172 @@ spec: type: object nullable: true type: array + dataPoolsMapped: + additionalProperties: + description: PoolSpec represents the spec of ceph pool + properties: + compressionMode: + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' + enum: + - none + - passive + - aggressive + - force + - "" + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid for CephFS + type: string + startTime: + description: StartTime indicates when to start the snapshot + type: string + type: object + type: array + type: object + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: MaxBytes represents the quota in bytes Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated storage pool, including the object itself (required for replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + description: The data pool settings with explicit data pool name + nullable: true + type: object metadataPool: description: The metadata pool settings nullable: true @@ -5801,6 +5967,7 @@ spec: x-kubernetes-preserve-unknown-fields: true required: - dataPools + - dataPoolsMapped - metadataPool - metadataServer type: object diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 103a11d8e91e0..12b8c9400832e 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -4970,6 +4970,172 @@ spec: type: object nullable: true type: array + dataPoolsMapped: + additionalProperties: + description: PoolSpec represents the spec of ceph pool + properties: + compressionMode: + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' + enum: + - none + - passive + - aggressive + - force + - "" + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid for CephFS + type: string + startTime: + description: StartTime indicates when to start the snapshot + type: string + type: object + type: array + type: object + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: MaxBytes represents the quota in bytes Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated storage pool, including the object itself (required for replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + description: The data pool settings with explicit data pool name + nullable: true + type: object metadataPool: description: The metadata pool settings nullable: true @@ -5798,6 +5964,7 @@ spec: x-kubernetes-preserve-unknown-fields: true required: - dataPools + - dataPoolsMapped - metadataPool - metadataServer type: object diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index e6f2e227a499e..9b5895d12a9d0 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -968,6 +968,10 @@ type FilesystemSpec struct { // +nullable DataPools []PoolSpec `json:"dataPools"` + // The data pool settings with explicit data pool name + // +nullable + DataPoolsMapped map[string]PoolSpec `json:"dataPoolsMapped"` + // Preserve pools on filesystem deletion // +optional PreservePoolsOnDelete bool `json:"preservePoolsOnDelete,omitempty"` diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 054381d12624c..19f50c48abf57 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -1940,6 +1940,13 @@ func (in *FilesystemSpec) DeepCopyInto(out *FilesystemSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DataPoolsMapped != nil { + in, out := &in.DataPoolsMapped, &out.DataPoolsMapped + *out = make(map[string]PoolSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } in.MetadataServer.DeepCopyInto(&out.MetadataServer) if in.Mirroring != nil { in, out := &in.Mirroring, &out.Mirroring diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index 2430c9b08f323..66a43ed0f854f 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -55,7 +55,7 @@ func createFilesystem( return err } - if len(fs.Spec.DataPools) != 0 { + if len(fs.Spec.DataPools) != 0 || len(fs.Spec.DataPoolsMapped) != 0 { f := newFS(fs.Name, fs.Namespace) if err := f.doFilesystemCreate(context, clusterInfo, clusterSpec, fs.Spec); err != nil { return errors.Wrapf(err, "failed to create filesystem %q", fs.Name) @@ -108,7 +108,7 @@ func deleteFilesystem( } // Permanently remove the filesystem if it was created by rook and the spec does not prevent it. - if len(fs.Spec.DataPools) != 0 && !fs.Spec.PreserveFilesystemOnDelete { + if (len(fs.Spec.DataPools) != 0 || len(fs.Spec.DataPoolsMapped) != 0) && !fs.Spec.PreserveFilesystemOnDelete { if err := cephclient.RemoveFilesystem(context, clusterInfo, fs.Name, fs.Spec.PreservePoolsOnDelete); err != nil { return errors.Wrapf(err, "failed to remove filesystem %q", fs.Name) } @@ -127,7 +127,7 @@ func validateFilesystem(context *clusterd.Context, clusterInfo *cephclient.Clust return errors.New("MetadataServer.ActiveCount must be at least 1") } // No data pool means that we expect the fs to exist already - if len(f.Spec.DataPools) == 0 { + if len(f.Spec.DataPools) == 0 && len(f.Spec.DataPoolsMapped) == 0 { return nil } if err := pool.ValidatePoolSpec(context, clusterInfo, clusterSpec, &f.Spec.MetadataPool); err != nil { @@ -139,6 +139,12 @@ func validateFilesystem(context *clusterd.Context, clusterInfo *cephclient.Clust return errors.Wrap(err, "Invalid data pool") } } + for namePool, p := range f.Spec.DataPoolsMapped { + localpoolSpec := p + if err := pool.ValidatePoolSpec(context, clusterInfo, clusterSpec, &localpoolSpec); err != nil { + return errors.Wrapf(err, "Invalid data pool '%s'", namePool) + } + } return nil } @@ -168,6 +174,14 @@ func SetPoolSize(f *Filesystem, context *clusterd.Context, clusterInfo *cephclie return errors.Wrapf(err, "failed to update datapool %q", poolName) } } + + for name, pool := range spec.DataPoolsMapped { + poolName := name + err := cephclient.CreatePoolWithProfile(context, clusterInfo, clusterSpec, poolName, pool, "") + if err != nil { + return errors.Wrapf(err, "failed to update datapool %q", poolName) + } + } return nil } @@ -193,6 +207,12 @@ func (f *Filesystem) updateFilesystem(context *clusterd.Context, clusterInfo *ce return err } } + + for name := range spec.DataPoolsMapped { + if err := cephclient.AddDataPoolToFilesystem(context, clusterInfo, f.Name, name); err != nil { + return err + } + } return nil } @@ -204,7 +224,7 @@ func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, clusterInfo * logger.Infof("filesystem %q already exists", f.Name) return f.updateFilesystem(context, clusterInfo, clusterSpec, spec) } - if len(spec.DataPools) == 0 { + if len(spec.DataPools) == 0 && len(spec.DataPoolsMapped) == 0 { return errors.New("at least one data pool must be specified") } @@ -256,6 +276,22 @@ func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, clusterInfo * } } + for name, pool := range spec.DataPoolsMapped { + poolName := name + if _, poolFound := reversedPoolMap[poolName]; !poolFound { + err = cephclient.CreatePoolWithProfile(context, clusterInfo, clusterSpec, poolName, pool, "") + if err != nil { + return errors.Wrapf(err, "failed to create data pool %q", poolName) + } + if pool.IsErasureCoded() { + // An erasure coded data pool used for a filesystem must allow overwrites + if err := cephclient.SetPoolProperty(context, clusterInfo, poolName, "allow_ec_overwrites", "true"); err != nil { + logger.Warningf("failed to set ec pool property. %v", err) + } + } + } + } + // create the filesystem ('fs new' needs to be forced in order to reuse pre-existing pools) // if only one pool is created new it won't work (to avoid inconsistencies). if err := cephclient.CreateFilesystem(context, clusterInfo, f.Name, metadataPoolName, dataPoolNames); err != nil { diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index b3a48845532f7..5475ad78fee62 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -63,6 +63,8 @@ func TestValidateSpec(t *testing.T) { assert.NotNil(t, validateFilesystem(context, clusterInfo, clusterSpec, fs)) p := cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}} fs.Spec.DataPools = append(fs.Spec.DataPools, p) + namedP := cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}} + fs.Spec.DataPoolsMapped = map[string]cephv1.PoolSpec{"named-pool": namedP} // missing metadata pool assert.NotNil(t, validateFilesystem(context, clusterInfo, clusterSpec, fs)) @@ -85,12 +87,20 @@ func isBasePoolOperation(fsName, command string, args []string) bool { return true } else if reflect.DeepEqual(args[0:7], []string{"osd", "pool", "create", fsName + "-data0", "0", "replicated", fsName + "-data0"}) { return true + } else if reflect.DeepEqual(args[0:7], []string{"osd", "pool", "create", "named-pool", "0", "replicated", "named-pool"}) { + return true } else if reflect.DeepEqual(args[0:5], []string{"osd", "crush", "rule", "create-replicated", fsName + "-data0"}) { return true + } else if reflect.DeepEqual(args[0:5], []string{"osd", "crush", "rule", "create-replicated", "named-pool"}) { + return true } else if reflect.DeepEqual(args[0:6], []string{"osd", "pool", "set", fsName + "-data0", "size", "1"}) { return true + } else if reflect.DeepEqual(args[0:6], []string{"osd", "pool", "set", "named-pool", "size", "1"}) { + return true } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, fsName + "-data0"}) { return true + } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, "named-pool"}) { + return true } return false } @@ -133,7 +143,7 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData } return string(createdFsResponse), nil } else if contains(args, "fs") && contains(args, "ls") { - return `[{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":4,"data_pool_ids":[5],"data_pools":["myfs-data0"]},{"name":"myfs2","metadata_pool":"myfs2-metadata","metadata_pool_id":6,"data_pool_ids":[7],"data_pools":["myfs2-data0"]},{"name":"leseb","metadata_pool":"cephfs.leseb.meta","metadata_pool_id":8,"data_pool_ids":[9],"data_pools":["cephfs.leseb.data"]}]`, nil + return `[{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":4,"data_pool_ids":[5,6],"data_pools":["myfs-data0","named-pool"]},{"name":"myfs2","metadata_pool":"myfs2-metadata","metadata_pool_id":7,"data_pool_ids":[8,9],"data_pools":["myfs2-data0","named-pool"]},{"name":"leseb","metadata_pool":"cephfs.leseb.meta","metadata_pool_id":10,"data_pool_ids":[11],"data_pools":["cephfs.leseb.data"]}]`, nil } else if contains(args, "fs") && contains(args, "dump") { return `{"standbys":[], "filesystems":[]}`, nil } else if contains(args, "osd") && contains(args, "lspools") { @@ -170,6 +180,16 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, fsName + "-data1"}) { *addDataOnePoolCount++ return "", nil + } else if reflect.DeepEqual(args[0:5], []string{"osd", "crush", "rule", "create-replicated", "named-pool-2"}) { + return "", nil + } else if reflect.DeepEqual(args[0:4], []string{"osd", "pool", "create", "named-pool-2"}) { + *createDataOnePoolCount++ + return "", nil + } else if reflect.DeepEqual(args[0:6], []string{"osd", "pool", "set", "named-pool-2", "size", "1"}) { + return "", nil + } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, "named-pool-2"}) { + *addDataOnePoolCount++ + return "", nil } else if contains(args, "versions") { versionStr, _ := json.Marshal( map[string]map[string]int{ @@ -233,6 +253,16 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, fsName + "-data1"}) { *addDataOnePoolCount++ return "", nil + } else if reflect.DeepEqual(args[0:5], []string{"osd", "crush", "rule", "create-replicated", "named-pool-2"}) { + return "", nil + } else if reflect.DeepEqual(args[0:4], []string{"osd", "pool", "create", "named-pool-2"}) { + *createDataOnePoolCount++ + return "", nil + } else if reflect.DeepEqual(args[0:6], []string{"osd", "pool", "set", "named-pool-2", "size", "1"}) { + return "", nil + } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, "named-pool-2"}) { + *addDataOnePoolCount++ + return "", nil } else if contains(args, "versions") { versionStr, _ := json.Marshal( map[string]map[string]int{ @@ -257,6 +287,9 @@ func fsTest(fsName string) cephv1.CephFilesystem { Spec: cephv1.FilesystemSpec{ MetadataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}, DataPools: []cephv1.PoolSpec{{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}}, + DataPoolsMapped: map[string]cephv1.PoolSpec{ + "named-pool": {Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}, + }, MetadataServer: cephv1.MetadataServerSpec{ ActiveCount: 1, Resources: v1.ResourceRequirements{ @@ -314,12 +347,13 @@ func TestCreateFilesystem(t *testing.T) { ConfigDir: configDir, Clientset: clientset} fs.Spec.DataPools = append(fs.Spec.DataPools, cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}) + fs.Spec.DataPoolsMapped["named-pool-2"] = cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}} err := createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.Nil(t, err) validateStart(ctx, t, context, fs) assert.ElementsMatch(t, []string{fmt.Sprintf("rook-ceph-mds-%s-a", fsName), fmt.Sprintf("rook-ceph-mds-%s-b", fsName)}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated)) - assert.Equal(t, 1, createDataOnePoolCount) - assert.Equal(t, 1, addDataOnePoolCount) + assert.Equal(t, 2, createDataOnePoolCount) + assert.Equal(t, 2, addDataOnePoolCount) testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) })