diff --git a/Documentation/ceph-monitoring.md b/Documentation/ceph-monitoring.md index 374845cd3e7b0..41078bf51cfc9 100644 --- a/Documentation/ceph-monitoring.md +++ b/Documentation/ceph-monitoring.md @@ -210,3 +210,30 @@ labels: prometheus: k8s [...] ``` + +### Horizontal Pod Scaling using Kubernetes Event-driven Autoscaling (KEDA) + +Using metrics exported from the Prometheus service, the horizontal pod scaling can use the custom metrics other than CPU and memory consumption. It can be done with help of Prometheus Scaler provided by the [KEDA](https://keda.sh/docs/2.4/scalers/prometheus/). See the [KEDA deployment guide](https://keda.sh/docs/2.4/deploy/) for details. + +Following is an example to autoscale RGW: +```YAML +apiVersion: keda.k8s.io/v1alpha1 +kind: ScaledObject +metadata: + name: rgw-scale + namespace: rook-ceph +spec: + scaleTargetRef: + kind: Deployment + deploymentName: rook-ceph-rgw-my-store-a # deployment for the autoscaling + minReplicaCount: 1 + maxReplicaCount: 5 + triggers: + - type: prometheus + metadata: + serverAddress: http://rook-prometheus.rook-ceph.svc:9090 + metricName: collecting_ceph_rgw_put + query: | + sum(rate(ceph_rgw_put[2m])) # promethues query used for autoscaling + threshold: "90" +``` diff --git a/Documentation/helm-ceph-cluster.md b/Documentation/helm-ceph-cluster.md index 07fc35a4a88e0..ff37077489be5 100644 --- a/Documentation/helm-ceph-cluster.md +++ b/Documentation/helm-ceph-cluster.md @@ -57,6 +57,7 @@ The following tables lists the configurable parameters of the rook-operator char | Parameter | Description | Default | | ---------------------- | -------------------------------------------------------------------- | ----------- | | `operatorNamespace` | Namespace of the Rook Operator | `rook-ceph` | +| `kubeVersion` | Optional override of the target kubernetes version | `` | | `configOverride` | Cluster ceph.conf override | | | `toolbox.enabled` | Enable Ceph debugging pod deployment. See [toolbox](ceph-toolbox.md) | `false` | | `toolbox.tolerations` | Toolbox tolerations | `[]` | diff --git a/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl b/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl index 8a7cf525d6a05..f7f5e2760a0ba 100644 --- a/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl +++ b/cluster/charts/rook-ceph-cluster/templates/_helpers.tpl @@ -31,3 +31,23 @@ Define the clusterName as defaulting to the release namespace {{- define "clusterName" -}} {{ .Values.clusterName | default .Release.Namespace }} {{- end -}} + +{{/* +Return the target Kubernetes version. +*/}} +{{- define "capabilities.kubeVersion" -}} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "capabilities.ingress.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} diff --git a/cluster/charts/rook-ceph-cluster/templates/ingress.yaml b/cluster/charts/rook-ceph-cluster/templates/ingress.yaml index d665de5f28fdf..078693288b932 100644 --- a/cluster/charts/rook-ceph-cluster/templates/ingress.yaml +++ b/cluster/charts/rook-ceph-cluster/templates/ingress.yaml @@ -1,12 +1,6 @@ {{- if .Values.ingress.dashboard.host }} --- -{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }} -apiVersion: networking.k8s.io/v1 -{{ else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} -apiVersion: networking.k8s.io/v1beta1 -{{ else }} -apiVersion: extensions/v1beta1 -{{ end -}} +apiVersion: {{ include "capabilities.ingress.apiVersion" . }} kind: Ingress metadata: name: {{ template "clusterName" . }}-dashboard @@ -20,7 +14,14 @@ spec: paths: - path: {{ .Values.ingress.dashboard.host.path | default "/" }} backend: -{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }} +{{- if (semverCompare "<1.18-0" (include "capabilities.kubeVersion" .)) }} + serviceName: rook-ceph-mgr-dashboard + {{- if .Values.cephClusterSpec.dashboard.ssl }} + servicePort: https-dashboard + {{- else }} + servicePort: http-dashboard + {{- end }} +{{- else }} service: name: rook-ceph-mgr-dashboard port: @@ -30,13 +31,6 @@ spec: name: http-dashboard {{- end }} pathType: Prefix -{{- else }} - serviceName: rook-ceph-mgr-dashboard - {{- if .Values.cephClusterSpec.dashboard.ssl }} - servicePort: https-dashboard - {{- else }} - servicePort: http-dashboard - {{- end }} {{- end }} {{- if .Values.ingress.dashboard.tls }} tls: {{- toYaml .Values.ingress.dashboard.tls | nindent 4 }} diff --git a/cluster/charts/rook-ceph-cluster/values.yaml b/cluster/charts/rook-ceph-cluster/values.yaml index 5a878fb941b37..48c221b4608e2 100644 --- a/cluster/charts/rook-ceph-cluster/values.yaml +++ b/cluster/charts/rook-ceph-cluster/values.yaml @@ -8,6 +8,9 @@ operatorNamespace: rook-ceph # The metadata.name of the CephCluster CR. The default name is the same as the namespace. # clusterName: rook-ceph +# Ability to override the kubernetes version used in rendering the helm chart +# kubeVersion: 1.21 + # Ability to override ceph.conf # configOverride: | # [global] diff --git a/cluster/examples/kubernetes/ceph/monitoring/keda-rgw.yaml b/cluster/examples/kubernetes/ceph/monitoring/keda-rgw.yaml new file mode 100644 index 0000000000000..de24eea2847b2 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/monitoring/keda-rgw.yaml @@ -0,0 +1,19 @@ +apiVersion: keda.k8s.io/v1alpha1 +kind: ScaledObject +metadata: + name: rgw-scale + namespace: rook-ceph +spec: + scaleTargetRef: + kind: Deployment + deploymentName: rook-ceph-rgw-my-store-a + minReplicaCount: 1 + maxReplicaCount: 5 + triggers: + - type: prometheus + metadata: + serverAddress: http://rook-prometheus.rook-ceph.svc:9090 + metricName: ceph_rgw_put_collector + query: | + sum(rate(ceph_rgw_put[2m])) + threshold: "90" diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index e980509dbe6fe..3b76261a51121 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -173,6 +173,11 @@ function validate_yaml() { kubectl create -f "${replication_url}/replication.storage.openshift.io_volumereplications.yaml" kubectl create -f "${replication_url}/replication.storage.openshift.io_volumereplicationclasses.yaml" + #create the KEDA CRDS + keda_version=2.4.0 + keda_url="https://github.com/kedacore/keda/releases/download/v${keda_version}/keda-${keda_version}.yaml" + kubectl apply -f "${keda_url}" + # skipping folders and some yamls that are only for openshift. manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc.yaml')" with_f_arg="$(echo "$manifests" | awk '{printf " -f %s",$1}')" # don't add newline