docs: add details about HPA via KEDA

Documentation/ceph-monitoring.md

@@ -210,3 +210,30 @@ labels:
     prometheus: k8s
 [...]
 ```
+
+### Horizontal Pod Scaling using Kubernetes Event-driven Autoscaling (KEDA)
+
+Using metrics exported from the Prometheus service, the horizontal pod scaling can use the custom metrics other than CPU and memory consumption. It can be done with help of Prometheus Scaler provided by the [KEDA](https://keda.sh/docs/2.4/scalers/prometheus/). See the [KEDA deployment guide](https://keda.sh/docs/2.4/deploy/) for details.
+
+Following is an example to autoscale RGW:
+```YAML
+apiVersion: keda.k8s.io/v1alpha1
+kind: ScaledObject
+metadata:
+ name: rgw-scale
+ namespace: rook-ceph
+spec:
+ scaleTargetRef:
+   kind: Deployment
+   deploymentName: rook-ceph-rgw-my-store-a # deployment for the autoscaling
+ minReplicaCount: 1
+ maxReplicaCount: 5
+ triggers:
+ - type: prometheus
+   metadata:
+     serverAddress: http://rook-prometheus.rook-ceph.svc:9090
+     metricName: collecting_ceph_rgw_put
+     query: |
+       sum(rate(ceph_rgw_put[2m])) # promethues query used for autoscaling
+     threshold: "90"
+```

cluster/examples/kubernetes/ceph/monitoring/keda-rgw.yaml

@@ -0,0 +1,19 @@
+apiVersion: keda.k8s.io/v1alpha1
+kind: ScaledObject
+metadata:
+  name: rgw-scale
+  namespace: rook-ceph
+spec:
+  scaleTargetRef:
+    kind: Deployment
+    deploymentName: rook-ceph-rgw-my-store-a
+  minReplicaCount: 1
+  maxReplicaCount: 5
+  triggers:
+    - type: prometheus
+      metadata:
+      serverAddress: http://rook-prometheus.rook-ceph.svc:9090
+      metricName: ceph_rgw_put_collector
+      query: |
+        sum(rate(ceph_rgw_put[2m]))
+      threshold: "90"

tests/scripts/github-action-helper.sh

@@ -173,6 +173,11 @@ function validate_yaml() {
   kubectl create -f "${replication_url}/replication.storage.openshift.io_volumereplications.yaml"
   kubectl create -f "${replication_url}/replication.storage.openshift.io_volumereplicationclasses.yaml"
 
+  #create the KEDA CRDS
+  keda_version=2.4.0
+  keda_url="https://github.com/kedacore/keda/releases/download/v${keda_version}/keda-${keda_version}.yaml"
+  kubectl apply -f "${keda_url}"
+
   # skipping folders and some yamls that are only for openshift.
   manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc*')"
   with_f_arg="$(echo "$manifests" | awk '{printf " -f %s",$1}')" # don't add newline