Merge pull request rook#9229 from rook/mergify/bp/release-1.7/pr-9202

docs: add details about HPA via KEDA (backport rook#9202)
Signed-off-by: Tom Hellier <me@tomhellier.com>

Documentation/ceph-monitoring.md

@@ -210,3 +210,30 @@ labels:
     prometheus: k8s
 [...]
 ```
+
+### Horizontal Pod Scaling using Kubernetes Event-driven Autoscaling (KEDA)
+
+Using metrics exported from the Prometheus service, the horizontal pod scaling can use the custom metrics other than CPU and memory consumption. It can be done with help of Prometheus Scaler provided by the [KEDA](https://keda.sh/docs/2.4/scalers/prometheus/). See the [KEDA deployment guide](https://keda.sh/docs/2.4/deploy/) for details.
+
+Following is an example to autoscale RGW:
+```YAML
+apiVersion: keda.k8s.io/v1alpha1
+kind: ScaledObject
+metadata:
+ name: rgw-scale
+ namespace: rook-ceph
+spec:
+ scaleTargetRef:
+   kind: Deployment
+   deploymentName: rook-ceph-rgw-my-store-a # deployment for the autoscaling
+ minReplicaCount: 1
+ maxReplicaCount: 5
+ triggers:
+ - type: prometheus
+   metadata:
+     serverAddress: http://rook-prometheus.rook-ceph.svc:9090
+     metricName: collecting_ceph_rgw_put
+     query: |
+       sum(rate(ceph_rgw_put[2m])) # promethues query used for autoscaling
+     threshold: "90"
+```

Documentation/helm-ceph-cluster.md

@@ -57,6 +57,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | Parameter              | Description                                                          | Default     |
 | ---------------------- | -------------------------------------------------------------------- | ----------- |
 | `operatorNamespace`    | Namespace of the Rook Operator                                       | `rook-ceph` |
+| `kubeVersion`          | Optional override of the target kubernetes version                   | ``          |
 | `configOverride`       | Cluster ceph.conf override                                           | <empty>     |
 | `toolbox.enabled`      | Enable Ceph debugging pod deployment. See [toolbox](ceph-toolbox.md) | `false`     |
 | `toolbox.tolerations`  | Toolbox tolerations                                                  | `[]`        |

cluster/charts/rook-ceph-cluster/templates/_helpers.tpl

@@ -31,3 +31,23 @@ Define the clusterName as defaulting to the release namespace
 {{- define "clusterName" -}}
 {{ .Values.clusterName | default .Release.Namespace }}
 {{- end -}}
+
+{{/*
+Return the target Kubernetes version.
+*/}}
+{{- define "capabilities.kubeVersion" -}}
+{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}}
+{{- end -}}
+
+{{/*
+Return the appropriate apiVersion for ingress.
+*/}}
+{{- define "capabilities.ingress.apiVersion" -}}
+{{- if semverCompare "<1.14-0" (include "capabilities.kubeVersion" .) -}}
+{{- print "extensions/v1beta1" -}}
+{{- else if semverCompare "<1.19-0" (include "capabilities.kubeVersion" .) -}}
+{{- print "networking.k8s.io/v1beta1" -}}
+{{- else -}}
+{{- print "networking.k8s.io/v1" -}}
+{{- end -}}
+{{- end -}}

cluster/charts/rook-ceph-cluster/templates/ingress.yaml

@@ -1,12 +1,6 @@
 {{- if .Values.ingress.dashboard.host }}
 ---
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
-apiVersion: networking.k8s.io/v1
-{{ else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
-apiVersion: networking.k8s.io/v1beta1
-{{ else }}
-apiVersion: extensions/v1beta1
-{{ end -}}
+apiVersion: {{ include "capabilities.ingress.apiVersion" . }}
 kind: Ingress
 metadata:
   name: {{ template "clusterName" . }}-dashboard
@@ -20,7 +14,14 @@ spec:
         paths:
           - path: {{ .Values.ingress.dashboard.host.path | default "/" }}
             backend:
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+{{- if (semverCompare "<1.18-0" (include "capabilities.kubeVersion" .)) }}
+              serviceName: rook-ceph-mgr-dashboard
+              {{- if .Values.cephClusterSpec.dashboard.ssl }}
+              servicePort: https-dashboard
+              {{- else }}
+              servicePort: http-dashboard
+              {{- end }}
+{{- else }}
               service:
                 name: rook-ceph-mgr-dashboard
                 port:
@@ -30,13 +31,6 @@ spec:
                   name: http-dashboard
                   {{- end }}
             pathType: Prefix
-{{- else }}
-              serviceName: rook-ceph-mgr-dashboard
-              {{- if .Values.cephClusterSpec.dashboard.ssl }}
-              servicePort: https-dashboard
-              {{- else }}
-              servicePort: http-dashboard
-              {{- end }}
 {{- end }}
   {{- if .Values.ingress.dashboard.tls }}
   tls: {{- toYaml .Values.ingress.dashboard.tls | nindent 4 }}

cluster/charts/rook-ceph-cluster/values.yaml

@@ -8,6 +8,9 @@ operatorNamespace: rook-ceph
 # The metadata.name of the CephCluster CR. The default name is the same as the namespace.
 # clusterName: rook-ceph
 
+# Ability to override the kubernetes version used in rendering the helm chart
+# kubeVersion: 1.21
+
 # Ability to override ceph.conf
 # configOverride: |
 #   [global]

cluster/examples/kubernetes/ceph/monitoring/keda-rgw.yaml

@@ -0,0 +1,19 @@
+apiVersion: keda.k8s.io/v1alpha1
+kind: ScaledObject
+metadata:
+  name: rgw-scale
+  namespace: rook-ceph
+spec:
+  scaleTargetRef:
+    kind: Deployment
+    deploymentName: rook-ceph-rgw-my-store-a
+  minReplicaCount: 1
+  maxReplicaCount: 5
+  triggers:
+    - type: prometheus
+      metadata:
+      serverAddress: http://rook-prometheus.rook-ceph.svc:9090
+      metricName: ceph_rgw_put_collector
+      query: |
+        sum(rate(ceph_rgw_put[2m]))
+      threshold: "90"

tests/scripts/github-action-helper.sh

@@ -173,6 +173,11 @@ function validate_yaml() {
   kubectl create -f "${replication_url}/replication.storage.openshift.io_volumereplications.yaml"
   kubectl create -f "${replication_url}/replication.storage.openshift.io_volumereplicationclasses.yaml"
 
+  #create the KEDA CRDS
+  keda_version=2.4.0
+  keda_url="https://github.com/kedacore/keda/releases/download/v${keda_version}/keda-${keda_version}.yaml"
+  kubectl apply -f "${keda_url}"
+
   # skipping folders and some yamls that are only for openshift.
   manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc.yaml')"
   with_f_arg="$(echo "$manifests" | awk '{printf " -f %s",$1}')" # don't add newline