From 16c27414efe2b7804686e357e579eeaca3849fb6 Mon Sep 17 00:00:00 2001 From: Denis Egorenko Date: Thu, 2 Dec 2021 16:34:54 +0400 Subject: [PATCH] file: allow to create CephFS data pools with predefined names Add an ability to create data pools for CephFS with predefined names. Related-Issue: rook#9295 Signed-off-by: Denis Egorenko --- Documentation/ceph-filesystem-crd.md | 37 ++-- Documentation/ceph-filesystem.md | 10 +- .../charts/rook-ceph/templates/resources.yaml | 174 +++++++++++++++++- deploy/examples/crds.yaml | 174 +++++++++++++++++- .../examples/csi/cephfs/storageclass-ec.yaml | 2 +- deploy/examples/csi/cephfs/storageclass.yaml | 2 +- deploy/examples/filesystem-ec.yaml | 16 +- deploy/examples/filesystem-test.yaml | 12 +- deploy/examples/filesystem.yaml | 32 ++-- pkg/apis/ceph.rook.io/v1/types.go | 19 +- .../ceph.rook.io/v1/zz_generated.deepcopy.go | 24 +++ pkg/operator/ceph/file/filesystem.go | 62 ++++--- pkg/operator/ceph/file/filesystem_test.go | 108 ++++++++++- 13 files changed, 583 insertions(+), 89 deletions(-) diff --git a/Documentation/ceph-filesystem-crd.md b/Documentation/ceph-filesystem-crd.md index f0a6a6d60ca37..da63d52e6027b 100644 --- a/Documentation/ceph-filesystem-crd.md +++ b/Documentation/ceph-filesystem-crd.md @@ -30,10 +30,12 @@ spec: failureDomain: host replicated: size: 3 - dataPools: - - failureDomain: host - replicated: - size: 3 + namedDataPools: + - name: replicated-pool + poolSpec: + failureDomain: host + replicated: + size: 3 preserveFilesystemOnDelete: true metadataServer: activeCount: 1 @@ -69,7 +71,7 @@ spec: ### Erasure Coded -Erasure coded pools require the OSDs to use `bluestore` for the configured [`storeType`](ceph-cluster-crd.md#osd-configuration-settings). Additionally, erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool. +Erasure coded pools require the OSDs to use `bluestore` for the configured [`storeType`](ceph-cluster-crd.md#osd-configuration-settings). Additionally, erasure coded pools can only be used with `namedDataPools`. The `metadataPool` must use a replicated pool. > **NOTE**: This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. @@ -85,10 +87,12 @@ spec: metadataPool: replicated: size: 3 - dataPools: - - erasureCoded: - dataChunks: 2 - codingChunks: 1 + namedDataPools: + - name: ec-pool + poolSpec: + erasureCoded: + dataChunks: 2 + codingChunks: 1 metadataServer: activeCount: 1 activeStandby: true @@ -117,10 +121,12 @@ spec: failureDomain: host replicated: size: 3 - dataPools: - - failureDomain: host - replicated: - size: 3 + namedDataPools: + - name: replicated-pool + poolSpec: + failureDomain: host + replicated: + size: 3 preserveFilesystemOnDelete: true metadataServer: activeCount: 1 @@ -183,7 +189,10 @@ See the official cephfs mirror documentation on [how to add a bootstrap peer](ht The pools allow all of the settings defined in the Pool CRD spec. For more details, see the [Pool CRD](ceph-pool-crd.md) settings. In the example above, there must be at least three hosts (size 3) and at least eight devices (6 data + 2 coding chunks) in the cluster. * `metadataPool`: The settings used to create the filesystem metadata pool. Must use replication. -* `dataPools`: The settings to create the filesystem data pools. If multiple pools are specified, Rook will add the pools to the filesystem. Assigning users or files to a pool is left as an exercise for the reader with the [CephFS documentation](http://docs.ceph.com/docs/master/cephfs/file-layouts/). The data pools can use replication or erasure coding. If erasure coding pools are specified, the cluster must be running with bluestore enabled on the OSDs. +* `namedDataPools`: The settings to create the filesystem data pools with predefined name. If multiple pools are specified, Rook will add the pools to the filesystem. Assigning users or files to a pool is left as an exercise for the reader with the [CephFS documentation](http://docs.ceph.com/docs/master/cephfs/file-layouts/). The data pools can use replication or erasure coding. If erasure coding pools are specified, the cluster must be running with bluestore enabled on the OSDs. Each element consist of: + * `name`: Name for pool + * `poolSpec`: Pool spec (ceph-pool-crd.md#spec) +* `dataPools`: (deprecated in favor of `namedDataPools`) The settings to create the filesystem data pools. If specified will be transitioned in runtime to `namedDataPools` with default name and index, thus pool name will be saved for any already present there pools. * `preserveFilesystemOnDelete`: If it is set to 'true' the filesystem will remain when the CephFilesystem resource is deleted. This is a security measure to avoid loss of data if the CephFilesystem resource is deleted accidentally. The default value is 'false'. This option diff --git a/Documentation/ceph-filesystem.md b/Documentation/ceph-filesystem.md index 59af1a91c1fcd..8880cf516ac9c 100644 --- a/Documentation/ceph-filesystem.md +++ b/Documentation/ceph-filesystem.md @@ -35,9 +35,11 @@ spec: metadataPool: replicated: size: 3 - dataPools: - - replicated: - size: 3 + namedDataPools: + - name: replicated-pool + poolSpec: + replicated: + size: 3 preserveFilesystemOnDelete: true metadataServer: activeCount: 1 @@ -98,7 +100,7 @@ parameters: # Ceph pool into which the volume shall be created # Required for provisionVolume: "true" - pool: myfs-data0 + pool: replicated-pool # The secrets contain Ceph admin credentials. These are generated automatically by the operator # in the same namespace as the cluster. diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 5923e746c23df..03bebd9cd30b5 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -4808,7 +4808,7 @@ spec: description: FilesystemSpec represents the spec of a file system properties: dataPools: - description: The data pool settings + description: (Deprecated in favor of namedDataPools) The data pool settings. items: description: PoolSpec represents the spec of ceph pool properties: @@ -4971,7 +4971,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object - nullable: true type: array metadataPool: description: The metadata pool settings @@ -5776,6 +5775,176 @@ spec: type: object type: array type: object + namedDataPools: + description: The data pool settings with explicit data pool name. Exactly one of dataPools or namedDataPools (preferred) must be specified. + items: + description: NamedPoolSpec represents the named ceph pool spec + properties: + compressionMode: + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' + enum: + - none + - passive + - aggressive + - force + - "" + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid for CephFS + type: string + startTime: + description: StartTime indicates when to start the snapshot + type: string + type: object + type: array + type: object + name: + description: Name of pool + type: string + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: MaxBytes represents the quota in bytes Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated storage pool, including the object itself (required for replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + type: array preserveFilesystemOnDelete: description: Preserve the fs in the cluster on CephFilesystem CR deletion. Setting this to true automatically implies PreservePoolsOnDelete is true. type: boolean @@ -5800,7 +5969,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true required: - - dataPools - metadataPool - metadataServer type: object diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 103a11d8e91e0..05c46e5c55f56 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -4805,7 +4805,7 @@ spec: description: FilesystemSpec represents the spec of a file system properties: dataPools: - description: The data pool settings + description: (Deprecated in favor of namedDataPools) The data pool settings. items: description: PoolSpec represents the spec of ceph pool properties: @@ -4968,7 +4968,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object - nullable: true type: array metadataPool: description: The metadata pool settings @@ -5773,6 +5772,176 @@ spec: type: object type: array type: object + namedDataPools: + description: The data pool settings with explicit data pool name. Exactly one of dataPools or namedDataPools (preferred) must be specified. + items: + description: NamedPoolSpec represents the named ceph pool spec + properties: + compressionMode: + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' + enum: + - none + - passive + - aggressive + - force + - "" + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. + minimum: 0 + type: integer + dataChunks: + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if available) - technically also any type in the crush map' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid for CephFS + type: string + startTime: + description: StartTime indicates when to start the snapshot + type: string + type: object + type: array + type: object + name: + description: Name of pool + type: string + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: MaxBytes represents the quota in bytes Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated storage pool, including the object itself (required for replicated pool type) + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + type: array preserveFilesystemOnDelete: description: Preserve the fs in the cluster on CephFilesystem CR deletion. Setting this to true automatically implies PreservePoolsOnDelete is true. type: boolean @@ -5797,7 +5966,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true required: - - dataPools - metadataPool - metadataServer type: object diff --git a/deploy/examples/csi/cephfs/storageclass-ec.yaml b/deploy/examples/csi/cephfs/storageclass-ec.yaml index 6c792812921a3..3173a2207c851 100644 --- a/deploy/examples/csi/cephfs/storageclass-ec.yaml +++ b/deploy/examples/csi/cephfs/storageclass-ec.yaml @@ -14,7 +14,7 @@ parameters: # Ceph pool into which the volume shall be created # Required for provisionVolume: "true" - pool: myfs-ec-data0 + pool: myfs-ec-pool # The secrets contain Ceph admin credentials. These are generated automatically by the operator # in the same namespace as the cluster. diff --git a/deploy/examples/csi/cephfs/storageclass.yaml b/deploy/examples/csi/cephfs/storageclass.yaml index 9b7c0ac7e62f2..7a4a09302b3d2 100644 --- a/deploy/examples/csi/cephfs/storageclass.yaml +++ b/deploy/examples/csi/cephfs/storageclass.yaml @@ -14,7 +14,7 @@ parameters: # Ceph pool into which the volume shall be created # Required for provisionVolume: "true" - pool: myfs-data0 + pool: myfs-replicated-pool # The secrets contain Ceph admin credentials. These are generated automatically by the operator # in the same namespace as the cluster. diff --git a/deploy/examples/filesystem-ec.yaml b/deploy/examples/filesystem-ec.yaml index 205adfd13a253..e64c62335f441 100644 --- a/deploy/examples/filesystem-ec.yaml +++ b/deploy/examples/filesystem-ec.yaml @@ -16,14 +16,16 @@ spec: # You need at least three OSDs on different nodes for this config to work size: 3 # The list of data pool specs - dataPools: + namedDataPools: # You need at least three `bluestore` OSDs on different nodes for this config to work - - erasureCoded: - dataChunks: 2 - codingChunks: 1 - # Inline compression mode for the data pool - parameters: - compression_mode: none + - name: myfs-ec-pool + poolSpec: + erasureCoded: + dataChunks: 2 + codingChunks: 1 + # Inline compression mode for the data pool + parameters: + compression_mode: none # Whether to preserve filesystem after CephFilesystem CRD deletion preserveFilesystemOnDelete: true # The metadata service (mds) configuration diff --git a/deploy/examples/filesystem-test.yaml b/deploy/examples/filesystem-test.yaml index c001f75312816..0a9b4ea1a3ba2 100644 --- a/deploy/examples/filesystem-test.yaml +++ b/deploy/examples/filesystem-test.yaml @@ -13,11 +13,13 @@ spec: replicated: size: 1 requireSafeReplicaSize: false - dataPools: - - failureDomain: osd - replicated: - size: 1 - requireSafeReplicaSize: false + namedDataPools: + - name: myfs-replicated-pool + poolSpec: + failureDomain: osd + replicated: + size: 1 + requireSafeReplicaSize: false preserveFilesystemOnDelete: false metadataServer: activeCount: 1 diff --git a/deploy/examples/filesystem.yaml b/deploy/examples/filesystem.yaml index eedd7181d8d93..35a383eba2395 100644 --- a/deploy/examples/filesystem.yaml +++ b/deploy/examples/filesystem.yaml @@ -24,21 +24,23 @@ spec: # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size #target_size_ratio: ".5" # The list of data pool specs. Can use replication or erasure coding. - dataPools: - - failureDomain: host - replicated: - size: 3 - # Disallow setting pool with replica 1, this could lead to data loss without recovery. - # Make sure you're *ABSOLUTELY CERTAIN* that is what you want - requireSafeReplicaSize: true - parameters: - # Inline compression mode for the data pool - # Further reference: https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression - compression_mode: - none - # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool - # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size - #target_size_ratio: ".5" + namedDataPools: + - name: myfs-replicated-pool + poolSpec: + failureDomain: host + replicated: + size: 3 + # Disallow setting pool with replica 1, this could lead to data loss without recovery. + # Make sure you're *ABSOLUTELY CERTAIN* that is what you want + requireSafeReplicaSize: true + parameters: + # Inline compression mode for the data pool + # Further reference: https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression + compression_mode: + none + # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool + # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size + #target_size_ratio: ".5" # Whether to preserve filesystem after CephFilesystem CRD deletion preserveFilesystemOnDelete: true # The metadata service (mds) configuration diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index e6f2e227a499e..804d204473546 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -639,6 +639,14 @@ type PoolSpec struct { Quotas QuotaSpec `json:"quotas,omitempty"` } +// NamedPoolSpec represents the named ceph pool spec +type NamedPoolSpec struct { + // Name of pool + Name string `json:"name"` + // PoolSpec represents the spec of ceph pool + PoolSpec `json:",inline"` +} + // MirrorHealthCheckSpec represents the health specification of a Ceph Storage Pool mirror type MirrorHealthCheckSpec struct { // +optional @@ -964,9 +972,14 @@ type FilesystemSpec struct { // +nullable MetadataPool PoolSpec `json:"metadataPool"` - // The data pool settings - // +nullable - DataPools []PoolSpec `json:"dataPools"` + // (Deprecated in favor of namedDataPools) The data pool settings. + // +optional + DataPools []PoolSpec `json:"dataPools,omitempty"` + + // The data pool settings with explicit data pool name. + // Exactly one of dataPools or namedDataPools (preferred) must be specified. + // +optional + NamedDataPools []NamedPoolSpec `json:"namedDataPools,omitempty"` // Preserve pools on filesystem deletion // +optional diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 054381d12624c..c4e4fb8c9dc02 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -1940,6 +1940,13 @@ func (in *FilesystemSpec) DeepCopyInto(out *FilesystemSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.NamedDataPools != nil { + in, out := &in.NamedDataPools, &out.NamedDataPools + *out = make([]NamedPoolSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.MetadataServer.DeepCopyInto(&out.MetadataServer) if in.Mirroring != nil { in, out := &in.Mirroring, &out.Mirroring @@ -2480,6 +2487,23 @@ func (in *NFSGaneshaSpec) DeepCopy() *NFSGaneshaSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamedPoolSpec) DeepCopyInto(out *NamedPoolSpec) { + *out = *in + in.PoolSpec.DeepCopyInto(&out.PoolSpec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamedPoolSpec. +func (in *NamedPoolSpec) DeepCopy() *NamedPoolSpec { + if in == nil { + return nil + } + out := new(NamedPoolSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { *out = *in diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index 2430c9b08f323..c8641fc59761d 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -55,7 +55,7 @@ func createFilesystem( return err } - if len(fs.Spec.DataPools) != 0 { + if len(fs.Spec.NamedDataPools) != 0 { f := newFS(fs.Name, fs.Namespace) if err := f.doFilesystemCreate(context, clusterInfo, clusterSpec, fs.Spec); err != nil { return errors.Wrapf(err, "failed to create filesystem %q", fs.Name) @@ -108,7 +108,7 @@ func deleteFilesystem( } // Permanently remove the filesystem if it was created by rook and the spec does not prevent it. - if len(fs.Spec.DataPools) != 0 && !fs.Spec.PreserveFilesystemOnDelete { + if (len(fs.Spec.DataPools) != 0 || len(fs.Spec.NamedDataPools) != 0) && !fs.Spec.PreserveFilesystemOnDelete { if err := cephclient.RemoveFilesystem(context, clusterInfo, fs.Name, fs.Spec.PreservePoolsOnDelete); err != nil { return errors.Wrapf(err, "failed to remove filesystem %q", fs.Name) } @@ -127,16 +127,31 @@ func validateFilesystem(context *clusterd.Context, clusterInfo *cephclient.Clust return errors.New("MetadataServer.ActiveCount must be at least 1") } // No data pool means that we expect the fs to exist already - if len(f.Spec.DataPools) == 0 { + if len(f.Spec.DataPools) == 0 && len(f.Spec.NamedDataPools) == 0 { return nil } + // Should be specified only one of DataPools or NamedDataPools + if len(f.Spec.DataPools) != 0 && len(f.Spec.NamedDataPools) != 0 { + return errors.New("Should be specified either DataPools, either NamedDataPools, but not both at the time.") + } if err := pool.ValidatePoolSpec(context, clusterInfo, clusterSpec, &f.Spec.MetadataPool); err != nil { return errors.Wrap(err, "invalid metadata pool") } - for _, p := range f.Spec.DataPools { - localpoolSpec := p + if len(f.Spec.DataPools) != 0 { + logger.Warning("Usage of DataPools is deprecated in favor of NamedDataPools") + f.Spec.NamedDataPools = make([]cephv1.NamedPoolSpec, len(f.Spec.DataPools)) + dataPoolNames := generateDataPoolNames(f.Name, f.Spec) + for idx, p := range f.Spec.DataPools { + f.Spec.NamedDataPools[idx] = cephv1.NamedPoolSpec{ + Name: dataPoolNames[idx], + PoolSpec: p, + } + } + } + for _, p := range f.Spec.NamedDataPools { + localpoolSpec := p.PoolSpec if err := pool.ValidatePoolSpec(context, clusterInfo, clusterSpec, &localpoolSpec); err != nil { - return errors.Wrap(err, "Invalid data pool") + return errors.Wrapf(err, "Invalid data pool '%s'", p.Name) } } @@ -159,11 +174,9 @@ func SetPoolSize(f *Filesystem, context *clusterd.Context, clusterInfo *cephclie if err != nil { return errors.Wrapf(err, "failed to update metadata pool %q", metadataPoolName) } - // generating the data pool's name - dataPoolNames := generateDataPoolNames(f, spec) - for i, pool := range spec.DataPools { - poolName := dataPoolNames[i] - err := cephclient.CreatePoolWithProfile(context, clusterInfo, clusterSpec, poolName, pool, "") + for _, pool := range spec.NamedDataPools { + poolName := pool.Name + err := cephclient.CreatePoolWithProfile(context, clusterInfo, clusterSpec, poolName, pool.PoolSpec, "") if err != nil { return errors.Wrapf(err, "failed to update datapool %q", poolName) } @@ -187,9 +200,8 @@ func (f *Filesystem) updateFilesystem(context *clusterd.Context, clusterInfo *ce return errors.Wrap(err, "failed to set pools size") } - dataPoolNames := generateDataPoolNames(f, spec) - for i := range spec.DataPools { - if err := cephclient.AddDataPoolToFilesystem(context, clusterInfo, f.Name, dataPoolNames[i]); err != nil { + for _, pool := range spec.NamedDataPools { + if err := cephclient.AddDataPoolToFilesystem(context, clusterInfo, f.Name, pool.Name); err != nil { return err } } @@ -204,7 +216,7 @@ func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, clusterInfo * logger.Infof("filesystem %q already exists", f.Name) return f.updateFilesystem(context, clusterInfo, clusterSpec, spec) } - if len(spec.DataPools) == 0 { + if len(spec.NamedDataPools) == 0 { return errors.New("at least one data pool must be specified") } @@ -239,17 +251,17 @@ func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, clusterInfo * } } - dataPoolNames := generateDataPoolNames(f, spec) - for i, pool := range spec.DataPools { - poolName := dataPoolNames[i] - if _, poolFound := reversedPoolMap[poolName]; !poolFound { - err = cephclient.CreatePoolWithProfile(context, clusterInfo, clusterSpec, poolName, pool, "") + dataPoolNames := make([]string, len(spec.NamedDataPools)) + for idx, pool := range spec.NamedDataPools { + dataPoolNames[idx] = pool.Name + if _, poolFound := reversedPoolMap[pool.Name]; !poolFound { + err = cephclient.CreatePoolWithProfile(context, clusterInfo, clusterSpec, pool.Name, pool.PoolSpec, "") if err != nil { - return errors.Wrapf(err, "failed to create data pool %q", poolName) + return errors.Wrapf(err, "failed to create data pool %q", pool.Name) } if pool.IsErasureCoded() { // An erasure coded data pool used for a filesystem must allow overwrites - if err := cephclient.SetPoolProperty(context, clusterInfo, poolName, "allow_ec_overwrites", "true"); err != nil { + if err := cephclient.SetPoolProperty(context, clusterInfo, pool.Name, "allow_ec_overwrites", "true"); err != nil { logger.Warningf("failed to set ec pool property. %v", err) } } @@ -277,11 +289,11 @@ func downFilesystem(context *clusterd.Context, clusterInfo *cephclient.ClusterIn return nil } -// generateDataPoolName generates DataPool name by prefixing the filesystem name to the constant DataPoolSuffix -func generateDataPoolNames(f *Filesystem, spec cephv1.FilesystemSpec) []string { +// generateDataPoolName generates DataPool (deprecated) name by prefixing the filesystem name to the constant DataPoolSuffix +func generateDataPoolNames(fsName string, spec cephv1.FilesystemSpec) []string { var dataPoolNames []string for i := range spec.DataPools { - poolName := fmt.Sprintf("%s-%s%d", f.Name, dataPoolSuffix, i) + poolName := fmt.Sprintf("%s-%s%d", fsName, dataPoolSuffix, i) dataPoolNames = append(dataPoolNames, poolName) } return dataPoolNames diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index b3a48845532f7..c696466e16fb4 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -63,6 +63,15 @@ func TestValidateSpec(t *testing.T) { assert.NotNil(t, validateFilesystem(context, clusterInfo, clusterSpec, fs)) p := cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}} fs.Spec.DataPools = append(fs.Spec.DataPools, p) + namedP := cephv1.NamedPoolSpec{ + Name: "named-pool", + PoolSpec: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}, + } + fs.Spec.NamedDataPools = append(fs.Spec.NamedDataPools, namedP) + + // both data pools (named and deprecated) are specified + assert.NotNil(t, validateFilesystem(context, clusterInfo, clusterSpec, fs)) + fs.Spec.DataPools = make([]cephv1.PoolSpec, 0) // missing metadata pool assert.NotNil(t, validateFilesystem(context, clusterInfo, clusterSpec, fs)) @@ -85,12 +94,20 @@ func isBasePoolOperation(fsName, command string, args []string) bool { return true } else if reflect.DeepEqual(args[0:7], []string{"osd", "pool", "create", fsName + "-data0", "0", "replicated", fsName + "-data0"}) { return true + } else if reflect.DeepEqual(args[0:7], []string{"osd", "pool", "create", "named-pool", "0", "replicated", "named-pool"}) { + return true } else if reflect.DeepEqual(args[0:5], []string{"osd", "crush", "rule", "create-replicated", fsName + "-data0"}) { return true + } else if reflect.DeepEqual(args[0:5], []string{"osd", "crush", "rule", "create-replicated", "named-pool"}) { + return true } else if reflect.DeepEqual(args[0:6], []string{"osd", "pool", "set", fsName + "-data0", "size", "1"}) { return true + } else if reflect.DeepEqual(args[0:6], []string{"osd", "pool", "set", "named-pool", "size", "1"}) { + return true } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, fsName + "-data0"}) { return true + } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, "named-pool"}) { + return true } return false } @@ -133,7 +150,7 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData } return string(createdFsResponse), nil } else if contains(args, "fs") && contains(args, "ls") { - return `[{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":4,"data_pool_ids":[5],"data_pools":["myfs-data0"]},{"name":"myfs2","metadata_pool":"myfs2-metadata","metadata_pool_id":6,"data_pool_ids":[7],"data_pools":["myfs2-data0"]},{"name":"leseb","metadata_pool":"cephfs.leseb.meta","metadata_pool_id":8,"data_pool_ids":[9],"data_pools":["cephfs.leseb.data"]}]`, nil + return `[{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":4,"data_pool_ids":[5,6],"data_pools":["myfs-data0","named-pool"]},{"name":"myfs2","metadata_pool":"myfs2-metadata","metadata_pool_id":7,"data_pool_ids":[8,9],"data_pools":["myfs2-data0","named-pool"]},{"name":"leseb","metadata_pool":"cephfs.leseb.meta","metadata_pool_id":10,"data_pool_ids":[11],"data_pools":["cephfs.leseb.data"]}]`, nil } else if contains(args, "fs") && contains(args, "dump") { return `{"standbys":[], "filesystems":[]}`, nil } else if contains(args, "osd") && contains(args, "lspools") { @@ -142,7 +159,7 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData return "", nil } else if isBasePoolOperation(fsName, command, args) { return "", nil - } else if reflect.DeepEqual(args[0:5], []string{"fs", "new", fsName, fsName + "-metadata", fsName + "-data0"}) { + } else if reflect.DeepEqual(args[0:5], []string{"fs", "new", fsName, fsName + "-metadata", "named-pool"}) { return "", nil } else if contains(args, "auth") && contains(args, "get-or-create-key") { return "{\"key\":\"mysecurekey\"}", nil @@ -160,14 +177,14 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData return "", nil } else if contains(args, "flag") && contains(args, "enable_multiple") { return "", nil - } else if reflect.DeepEqual(args[0:5], []string{"osd", "crush", "rule", "create-replicated", fsName + "-data1"}) { + } else if reflect.DeepEqual(args[0:5], []string{"osd", "crush", "rule", "create-replicated", "named-pool-2"}) { return "", nil - } else if reflect.DeepEqual(args[0:4], []string{"osd", "pool", "create", fsName + "-data1"}) { + } else if reflect.DeepEqual(args[0:4], []string{"osd", "pool", "create", "named-pool-2"}) { *createDataOnePoolCount++ return "", nil - } else if reflect.DeepEqual(args[0:6], []string{"osd", "pool", "set", fsName + "-data1", "size", "1"}) { + } else if reflect.DeepEqual(args[0:6], []string{"osd", "pool", "set", "named-pool-2", "size", "1"}) { return "", nil - } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, fsName + "-data1"}) { + } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, "named-pool-2"}) { *addDataOnePoolCount++ return "", nil } else if contains(args, "versions") { @@ -209,6 +226,8 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData return "", nil } else if reflect.DeepEqual(args[0:5], []string{"fs", "new", fsName, fsName + "-metadata", fsName + "-data0"}) { return "", nil + } else if reflect.DeepEqual(args[0:5], []string{"fs", "new", fsName, fsName + "-metadata", "named-pool"}) { + return "", nil } else if contains(args, "auth") && contains(args, "get-or-create-key") { return "{\"key\":\"mysecurekey\"}", nil } else if contains(args, "auth") && contains(args, "del") { @@ -233,6 +252,16 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, fsName + "-data1"}) { *addDataOnePoolCount++ return "", nil + } else if reflect.DeepEqual(args[0:5], []string{"osd", "crush", "rule", "create-replicated", "named-pool-2"}) { + return "", nil + } else if reflect.DeepEqual(args[0:4], []string{"osd", "pool", "create", "named-pool-2"}) { + *createDataOnePoolCount++ + return "", nil + } else if reflect.DeepEqual(args[0:6], []string{"osd", "pool", "set", "named-pool-2", "size", "1"}) { + return "", nil + } else if reflect.DeepEqual(args[0:4], []string{"fs", "add_data_pool", fsName, "named-pool-2"}) { + *addDataOnePoolCount++ + return "", nil } else if contains(args, "versions") { versionStr, _ := json.Marshal( map[string]map[string]int{ @@ -252,6 +281,32 @@ func fsExecutor(t *testing.T, fsName, configDir string, multiFS bool, createData } func fsTest(fsName string) cephv1.CephFilesystem { + return cephv1.CephFilesystem{ + ObjectMeta: metav1.ObjectMeta{Name: fsName, Namespace: "ns"}, + Spec: cephv1.FilesystemSpec{ + MetadataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}, + NamedDataPools: []cephv1.NamedPoolSpec{ + { + Name: "named-pool", + PoolSpec: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}, + }, + }, + MetadataServer: cephv1.MetadataServerSpec{ + ActiveCount: 1, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(4294967296, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), + }, + }, + }, + }, + } +} + +func fsTestWithDeprecatedPools(fsName string) cephv1.CephFilesystem { return cephv1.CephFilesystem{ ObjectMeta: metav1.ObjectMeta{Name: fsName, Namespace: "ns"}, Spec: cephv1.FilesystemSpec{ @@ -313,7 +368,10 @@ func TestCreateFilesystem(t *testing.T) { Executor: executor, ConfigDir: configDir, Clientset: clientset} - fs.Spec.DataPools = append(fs.Spec.DataPools, cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}) + fs.Spec.NamedDataPools = append(fs.Spec.NamedDataPools, cephv1.NamedPoolSpec{ + Name: "named-pool-2", + PoolSpec: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}, + }) err := createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.Nil(t, err) validateStart(ctx, t, context, fs) @@ -323,6 +381,40 @@ func TestCreateFilesystem(t *testing.T) { testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) }) + t.Run("start basic filesystem with deprecated dataPools section and increase data pools", func(t *testing.T) { + fsNameForDeprecated := fsName + "-deprecatedStuff" + fsDeprecated := fsTestWithDeprecatedPools(fsNameForDeprecated) + createDataOnePoolCount = 0 + addDataOnePoolCount = 0 + executor := fsExecutor(t, fsNameForDeprecated, configDir, false, &createDataOnePoolCount, &addDataOnePoolCount) + context = &clusterd.Context{ + Executor: executor, + ConfigDir: configDir, + Clientset: clientset} + // give validation to update spec in runtime + assert.Nil(t, validateFilesystem(context, clusterInfo, &cephv1.ClusterSpec{}, &fsDeprecated)) + // start a basic cluster + err := createFilesystem(context, clusterInfo, fsDeprecated, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") + assert.Nil(t, err) + validateStart(ctx, t, context, fsDeprecated) + testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) + // new reconcile start for data pools update + context = &clusterd.Context{ + Executor: executor, + ConfigDir: configDir, + Clientset: clientset} + fsDeprecated = fsTestWithDeprecatedPools(fsNameForDeprecated) + fsDeprecated.Spec.DataPools = append(fsDeprecated.Spec.DataPools, cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}) + // give validation to update spec in runtime + assert.Nil(t, validateFilesystem(context, clusterInfo, &cephv1.ClusterSpec{}, &fsDeprecated)) + err = createFilesystem(context, clusterInfo, fsDeprecated, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") + assert.Nil(t, err) + validateStart(ctx, t, context, fsDeprecated) + assert.Equal(t, 1, createDataOnePoolCount) + assert.Equal(t, 1, addDataOnePoolCount) + testopk8s.ClearDeploymentsUpdated(deploymentsUpdated) + }) + t.Run("multiple filesystem creation", func(t *testing.T) { context = &clusterd.Context{ Executor: fsExecutor(t, fsName, configDir, true, &createDataOnePoolCount, &addDataOnePoolCount), @@ -424,7 +516,7 @@ func TestUpgradeFilesystem(t *testing.T) { return "", errors.New("fail mds failed") } else if isBasePoolOperation(fsName, command, args) { return "", nil - } else if reflect.DeepEqual(args[0:5], []string{"fs", "new", fsName, fsName + "-metadata", fsName + "-data0"}) { + } else if reflect.DeepEqual(args[0:5], []string{"fs", "new", fsName, fsName + "-metadata", "named-pool"}) { return "", nil } else if contains(args, "auth") && contains(args, "get-or-create-key") { return "{\"key\":\"mysecurekey\"}", nil