Fix queries if node class label is added (#99)

Prometheus keeps series "alive" for 5 minutes after they get dropped from a scrape target. This leads to a slight overlap in the `kube_node_labels` metric if a node is relabelled.

This commit prioritises series having a node class label over series without.

This still does not work if a node class label changes from non-empty to non-empty but that should never happen. I don't think there is a generic way to handle this case.

Tested with LPG2 data at `2022-11-11 10:00:00Z`.

pkg/db/seeds/appuio_cloud_memory.promql

@@ -16,25 +16,32 @@ sum_over_time(
               (
                 # Select used memory if higher.
                 (
-                  sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!=""} * on(node) group_left(label_appuio_io_node_class) kube_node_labels)
+                  sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!=""}
+                    * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""}))
                   # IMPORTANT: one clause must use equal. If used grater and lesser than, equal values will be dropped.
                   >=
-                  sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} * on(node) group_left(label_appuio_io_node_class) kube_node_labels)
+                  sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"}
+                    * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"}
+                    * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""}))
                 )
                 or
                 # Select reserved memory if higher.
                 (
                   # IMPORTANT: The desired time series must always be first.
-                  sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} * on(node) group_left(label_appuio_io_node_class) kube_node_labels)
+                  sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"}
+                    * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"}
+                    * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""}))
                   >
-                  sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!=""} * on(node) group_left(label_appuio_io_node_class) kube_node_labels)
+                  sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!=""}
+                    * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""}))
                 )
               )
               # Add CPU requests in violation to the ratio provided by the platform.
               + clamp_min(
                   # Convert CPU request to their memory equivalent.
                   sum by(cluster_id, namespace, label_appuio_io_node_class) (
-                    kube_pod_container_resource_requests{resource="cpu"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} * on(node) group_left(label_appuio_io_node_class) kube_node_labels
+                    kube_pod_container_resource_requests{resource="cpu"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"}
+                      * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""})
                     # Build that ratio from static values
                     * on(cluster_id) group_left()(
                       # Build a time series of ratio for Cloudscale LPG 2 (4096 MiB/core)
@@ -44,7 +51,8 @@ sum_over_time(
                     )
                   )
                   # Subtract memory request
-                  - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} * on(node) group_left(label_appuio_io_node_class) kube_node_labels
+                  - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"}
+                    * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""})
               # Only values above zero are in violation.
               ), 0)
             )

pkg/db/seeds/appuio_cloud_memory_sub_cpu.promql

@@ -13,7 +13,8 @@ sum_over_time(
             (
               sum by(cluster_id, namespace, label_appuio_io_node_class) (
                 # Get the CPU requests
-                kube_pod_container_resource_requests{resource="cpu"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} * on(node) group_left(label_appuio_io_node_class) kube_node_labels
+                kube_pod_container_resource_requests{resource="cpu"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"}
+                  * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""})
                 # Convert them to their memory equivalent by multiplying them by the memory to CPU ratio
                 # Build that ratio from static values
                 * on(cluster_id) group_left()(
@@ -23,7 +24,8 @@ sum_over_time(
                   or label_replace(vector(5333057536), "cluster_id", "c-appuio-exoscale-ch-gva-2-0", "", "")
                 )
               )
-              - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} * on(node) group_left(label_appuio_io_node_class) kube_node_labels)
+              - sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"}
+                * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""}))
             )
             *
             # Join namespace label `label_appuio_io_organization` as `tenant_id`.

pkg/db/seeds/appuio_cloud_memory_sub_memory.promql

@@ -12,10 +12,13 @@ sum_over_time(
           clamp_min(
             (
               clamp_min(
-                sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"} * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"} * on(node) group_left(label_appuio_io_node_class) kube_node_labels),
+                sum by(cluster_id, namespace, label_appuio_io_node_class) (kube_pod_container_resource_requests{resource="memory"}
+                  * on(uid, cluster_id, pod, namespace) group_left kube_pod_status_phase{phase="Running"}
+                  * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""})),
                 128 * 1024 * 1024
               )
-              - sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!=""} * on(node) group_left(label_appuio_io_node_class) kube_node_labels)
+              - sum by(cluster_id, namespace, label_appuio_io_node_class) (container_memory_working_set_bytes{image!=""}
+                * on(node) group_left(label_appuio_io_node_class) (kube_node_labels{label_appuio_io_node_class!=""} or on(node) kube_node_labels{label_appuio_io_node_class=""}))
             ),
             0
           )