From 6649f758bd12e77df6d8c38c5c2e850c35d6a060 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Mon, 27 Sep 2021 15:50:11 -0600 Subject: [PATCH] csi: no longer install the volumereplication crds from rook The volume replication CRDs are an external component, not owned by Rook. Therefore, they should be installed as any other independent component in case the admin will install other consumers of the volumereplication CRDs in the future in addition to Rook and the CSI driver. Signed-off-by: Travis Nielsen --- Documentation/ceph-csi-drivers.md | 20 +- build/crds/build-crds.sh | 7 - .../charts/rook-ceph/templates/resources.yaml | 222 ------------------ cluster/charts/rook-ceph/values.yaml | 4 +- cluster/examples/kubernetes/ceph/crds.yaml | 220 ----------------- .../kubernetes/ceph/operator-openshift.yaml | 4 +- .../examples/kubernetes/ceph/operator.yaml | 4 +- tests/framework/installer/ceph_installer.go | 24 ++ tests/framework/installer/settings.go | 4 + tests/integration/ceph_object_test.go | 4 +- tests/integration/ceph_smoke_test.go | 4 +- 11 files changed, 54 insertions(+), 463 deletions(-) diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/ceph-csi-drivers.md index 9c216d00e9175..ab9ad1ffe7176 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/ceph-csi-drivers.md @@ -77,8 +77,18 @@ PVC will be updated to new size. ## RBD Mirroring To support RBD Mirroring, the [Volume Replication Operator](https://github.com/csi-addons/volume-replication-operator/blob/main/README.md) will be started in the RBD provisioner pod. -Volume Replication Operator is a kubernetes operator that provides common and reusable APIs for storage disaster recovery. It is based on [csi-addons/spec](https://github.com/csi-addons/spec) specification and can be used by any storage provider. -It follows controller pattern and provides extended APIs for storage disaster recovery. The extended APIs are provided via Custom Resource Definition (CRD). -To enable volume replication: -- For Helm deployments see the [helm settings](helm-operator.md#configuration). -- For non-Helm deployments set `CSI_ENABLE_VOLUME_REPLICATION: "true"` in the operator.yaml +The Volume Replication Operator is a kubernetes operator that provides common and reusable APIs for storage disaster recovery. It is based on [csi-addons/spec](https://github.com/csi-addons/spec) specification and can be used by any storage provider. +It follows the controller pattern and provides extended APIs for storage disaster recovery. The extended APIs are provided via Custom Resource Definitions (CRDs). + +### Enable volume replication + +1. Install the volume replication CRDs: + +```console +kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/main/config/crd/bases/replication.storage.openshift.io_volumereplications.yaml +kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/main/config/crd/bases/replication.storage.openshift.io_volumereplicationclasses.yaml +``` + +2. Enable the volume replication controller: + - For Helm deployments see the [csi.volumeReplication.enabled setting](helm-operator.md#configuration). + - For non-Helm deployments set `CSI_ENABLE_VOLUME_REPLICATION: "true"` in operator.yaml diff --git a/build/crds/build-crds.sh b/build/crds/build-crds.sh index 74d3173babb6b..71ea95955503a 100755 --- a/build/crds/build-crds.sh +++ b/build/crds/build-crds.sh @@ -53,11 +53,6 @@ generating_crds_v1() { $YQ_BIN_PATH w -i "${OLM_CATALOG_DIR}"/ceph.rook.io_cephclusters.yaml spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.storage.properties.storageClassDeviceSets.items.properties.volumeClaimTemplates.items.properties.metadata.properties.annotations.x-kubernetes-preserve-unknown-fields true } -generate_vol_rep_crds() { - echo "Generating volume replication crds in crds.yaml" - "$CONTROLLER_GEN_BIN_PATH" "$CRD_OPTIONS" paths="github.com/csi-addons/volume-replication-operator/api/v1alpha1" output:crd:artifacts:config="$OLM_CATALOG_DIR" -} - generating_main_crd() { true > "$CEPH_CRDS_FILE_PATH" true > "$CEPH_HELM_CRDS_FILE_PATH" @@ -94,8 +89,6 @@ if [ -z "$NO_OB_OBC_VOL_GEN" ]; then copy_ob_obc_crds fi -generate_vol_rep_crds - generating_main_crd for crd in "$OLM_CATALOG_DIR/"*.yaml; do diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 5e1bf1caa132a..a62ecad5dc106 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -8945,227 +8945,5 @@ spec: x-kubernetes-preserve-unknown-fields: true subresources: status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c - helm.sh/resource-policy: keep - creationTimestamp: null - name: volumereplicationclasses.replication.storage.openshift.io -spec: - group: replication.storage.openshift.io - names: - kind: VolumeReplicationClass - listKind: VolumeReplicationClassList - plural: volumereplicationclasses - shortNames: - - vrc - singular: volumereplicationclass - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.provisioner - name: provisioner - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: VolumeReplicationClass is the Schema for the volumereplicationclasses API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: VolumeReplicationClassSpec specifies parameters that an underlying storage system uses when creating a volume replica. A specific VolumeReplicationClass is used by specifying its name in a VolumeReplication object. - properties: - parameters: - additionalProperties: - type: string - description: Parameters is a key-value map with storage provisioner specific configurations for creating volume replicas - type: object - provisioner: - description: Provisioner is the name of storage provisioner - type: string - required: - - provisioner - type: object - status: - description: VolumeReplicationClassStatus defines the observed state of VolumeReplicationClass - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c - helm.sh/resource-policy: keep - creationTimestamp: null - name: volumereplications.replication.storage.openshift.io -spec: - group: replication.storage.openshift.io - names: - kind: VolumeReplication - listKind: VolumeReplicationList - plural: volumereplications - shortNames: - - vr - singular: volumereplication - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .spec.volumeReplicationClass - name: volumeReplicationClass - type: string - - jsonPath: .spec.dataSource.name - name: pvcName - type: string - - jsonPath: .spec.replicationState - name: desiredState - type: string - - jsonPath: .status.state - name: currentState - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: VolumeReplication is the Schema for the volumereplications API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: VolumeReplicationSpec defines the desired state of VolumeReplication - properties: - dataSource: - description: DataSource represents the object associated with the volume - properties: - apiGroup: - description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - replicationState: - description: ReplicationState represents the replication operation to be performed on the volume. Supported operations are "primary", "secondary" and "resync" - enum: - - primary - - secondary - - resync - type: string - volumeReplicationClass: - description: VolumeReplicationClass is the VolumeReplicationClass name for this VolumeReplication resource - type: string - required: - - dataSource - - replicationState - - volumeReplicationClass - type: object - status: - description: VolumeReplicationStatus defines the observed state of VolumeReplication - properties: - conditions: - description: Conditions are the list of conditions and their status. - items: - description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - lastCompletionTime: - format: date-time - type: string - lastStartTime: - format: date-time - type: string - message: - type: string - observedGeneration: - description: observedGeneration is the last generation change the operator has dealt with - format: int64 - type: integer - state: - description: State captures the latest state of the replication operation - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] {{- end }} diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 573d92c228fbe..eada379382d50 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -288,7 +288,9 @@ csi: #cephfsPodLabels: "key1=value1,key2=value2" # Labels to add to the CSI RBD Deployments and DaemonSets Pods. #rbdPodLabels: "key1=value1,key2=value2" - # Enable volume replication controller + # Enable the volume replication controller. + # Before enabling, ensure the Volume Replication CRDs are created. + # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring volumeReplication: enabled: false #image: "quay.io/csiaddons/volumereplication-operator:v0.1.0" diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 0fb957cf76d9f..32c54e21c619e 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -8933,223 +8933,3 @@ spec: x-kubernetes-preserve-unknown-fields: true subresources: status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c - creationTimestamp: null - name: volumereplicationclasses.replication.storage.openshift.io -spec: - group: replication.storage.openshift.io - names: - kind: VolumeReplicationClass - listKind: VolumeReplicationClassList - plural: volumereplicationclasses - shortNames: - - vrc - singular: volumereplicationclass - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.provisioner - name: provisioner - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: VolumeReplicationClass is the Schema for the volumereplicationclasses API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: VolumeReplicationClassSpec specifies parameters that an underlying storage system uses when creating a volume replica. A specific VolumeReplicationClass is used by specifying its name in a VolumeReplication object. - properties: - parameters: - additionalProperties: - type: string - description: Parameters is a key-value map with storage provisioner specific configurations for creating volume replicas - type: object - provisioner: - description: Provisioner is the name of storage provisioner - type: string - required: - - provisioner - type: object - status: - description: VolumeReplicationClassStatus defines the observed state of VolumeReplicationClass - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c - creationTimestamp: null - name: volumereplications.replication.storage.openshift.io -spec: - group: replication.storage.openshift.io - names: - kind: VolumeReplication - listKind: VolumeReplicationList - plural: volumereplications - shortNames: - - vr - singular: volumereplication - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .spec.volumeReplicationClass - name: volumeReplicationClass - type: string - - jsonPath: .spec.dataSource.name - name: pvcName - type: string - - jsonPath: .spec.replicationState - name: desiredState - type: string - - jsonPath: .status.state - name: currentState - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: VolumeReplication is the Schema for the volumereplications API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: VolumeReplicationSpec defines the desired state of VolumeReplication - properties: - dataSource: - description: DataSource represents the object associated with the volume - properties: - apiGroup: - description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - replicationState: - description: ReplicationState represents the replication operation to be performed on the volume. Supported operations are "primary", "secondary" and "resync" - enum: - - primary - - secondary - - resync - type: string - volumeReplicationClass: - description: VolumeReplicationClass is the VolumeReplicationClass name for this VolumeReplication resource - type: string - required: - - dataSource - - replicationState - - volumeReplicationClass - type: object - status: - description: VolumeReplicationStatus defines the observed state of VolumeReplication - properties: - conditions: - description: Conditions are the list of conditions and their status. - items: - description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - lastCompletionTime: - format: date-time - type: string - lastStartTime: - format: date-time - type: string - message: - type: string - observedGeneration: - description: observedGeneration is the last generation change the operator has dealt with - format: int64 - type: integer - state: - description: State captures the latest state of the replication operation - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 512f0d17c028c..5d6c374333cca 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -402,7 +402,9 @@ data: # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. ROOK_ENABLE_DISCOVERY_DAEMON: "false" - # Enable volume replication controller + # Enable the volume replication controller + # Before enabling, ensure the Volume Replication CRDs are created. + # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring CSI_ENABLE_VOLUME_REPLICATION: "false" # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15. ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index b95f4c387fa07..b79a963144701 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -326,7 +326,9 @@ data: ROOK_ENABLE_DISCOVERY_DAEMON: "false" # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15. ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15" - # Enable volume replication controller + # Enable the volume replication controller. + # Before enabling, ensure the Volume Replication CRDs are created. + # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring CSI_ENABLE_VOLUME_REPLICATION: "false" # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0" --- diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 5ca339644d3af..39a38d1097562 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -64,6 +64,9 @@ const ( osd_pool_default_size = 1 bdev_flock_retry = 20 ` + volumeReplicationBaseURL = "https://raw.githubusercontent.com/csi-addons/volume-replication-operator/main/config/crd/bases/" + volumeReplicationCRDURL = volumeReplicationBaseURL + "replication.storage.openshift.io_volumereplications.yaml" + volumeReplicationClassCRDURL = volumeReplicationBaseURL + "replication.storage.openshift.io_volumereplicationclasses.yaml" ) var ( @@ -130,6 +133,10 @@ func (h *CephInstaller) CreateCephOperator() (err error) { return errors.Errorf("Failed to start admission controllers: %v", err) } + if err := h.CreateVolumeReplicationCRDs(); err != nil { + return errors.Wrap(err, "failed to create volume replication CRDs") + } + _, err = h.k8shelper.KubectlWithStdin(h.Manifests.GetOperator(), createFromStdinArgs...) if err != nil { return errors.Errorf("Failed to create rook-operator pod: %v", err) @@ -139,6 +146,23 @@ func (h *CephInstaller) CreateCephOperator() (err error) { return nil } +func (h *CephInstaller) CreateVolumeReplicationCRDs() (err error) { + if !h.Manifests.Settings().EnableVolumeReplication { + logger.Info("volume replication CRDs skipped") + return nil + } + + logger.Info("Creating volume replication CRDs") + if _, err := h.k8shelper.KubectlWithStdin(readManifestFromURL(volumeReplicationCRDURL), createFromStdinArgs...); err != nil { + return errors.Wrap(err, "failed to create volumereplication CRD") + } + + if _, err := h.k8shelper.KubectlWithStdin(readManifestFromURL(volumeReplicationClassCRDURL), createFromStdinArgs...); err != nil { + return errors.Wrap(err, "failed to create volumereplicationclass CRD") + } + return nil +} + func (h *CephInstaller) startAdmissionController() error { if !h.k8shelper.VersionAtLeast("v1.16.0") { logger.Info("skipping the admission controller on K8s version older than v1.16") diff --git a/tests/framework/installer/settings.go b/tests/framework/installer/settings.go index 7bb7a4c2339f2..4b268b2fbf09e 100644 --- a/tests/framework/installer/settings.go +++ b/tests/framework/installer/settings.go @@ -46,6 +46,10 @@ func readManifest(provider, filename string) string { func readManifestFromGithub(rookVersion, provider, filename string) string { url := fmt.Sprintf("https://raw.githubusercontent.com/rook/rook/%s/cluster/examples/kubernetes/%s/%s", rookVersion, provider, filename) + return readManifestFromURL(url) +} + +func readManifestFromURL(url string) string { logger.Infof("Retrieving manifest: %s", url) var response *http.Response var err error diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go index 8099513660b5f..75ca51a202083 100644 --- a/tests/integration/ceph_object_test.go +++ b/tests/integration/ceph_object_test.go @@ -74,14 +74,12 @@ func (s *ObjectSuite) SetupSuite() { SkipOSDCreation: false, EnableAdmissionController: true, UseCrashPruner: true, + EnableVolumeReplication: true, RookVersion: installer.LocalBuildTag, CephVersion: installer.ReturnCephVersion(), } s.settings.ApplyEnvVars() s.installer, s.k8sh = StartTestCluster(s.T, s.settings) - if s.k8sh.VersionAtLeast("v1.16.0") { - s.settings.EnableVolumeReplication = true - } s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests) } diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index 63e8f8e7f1038..d05f5873e1b45 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -93,14 +93,12 @@ func (s *SmokeSuite) SetupSuite() { SkipOSDCreation: false, EnableAdmissionController: true, UseCrashPruner: true, + EnableVolumeReplication: true, RookVersion: installer.LocalBuildTag, CephVersion: installer.ReturnCephVersion(), } s.settings.ApplyEnvVars() s.installer, s.k8sh = StartTestCluster(s.T, s.settings) - if s.k8sh.VersionAtLeast("v1.16.0") { - s.settings.EnableVolumeReplication = true - } s.helper = clients.CreateTestClient(s.k8sh, s.installer.Manifests) }