Fix query on namespace label changes and organization change

Changes to namespace labels and organization changes can lead to many-to-many matching errors

pkg/db/seeds/appuio_cloud_loadbalancer.promql

@@ -12,11 +12,17 @@ sum_over_time(
         # Join the namespace label to get the tenant
         on(cluster_id, namespace)
         group_left(tenant_id)
-        label_replace(
-          kube_namespace_labels{label_appuio_io_organization=~".+"},
-          "tenant_id",
-          "$1",
-          "label_appuio_io_organization", "(.*)"
+        (
+          bottomk(1,
+            min by (cluster_id, namespace, tenant_id) (
+              label_replace(
+                kube_namespace_labels{label_appuio_io_organization=~".+"},
+                "tenant_id",
+                "$1",
+                "label_appuio_io_organization", "(.*)"
+              )
+            )
+          ) by(cluster_id, namespace)
         ),
         "product",
         "appuio_cloud_loadbalancer",

pkg/db/seeds/appuio_cloud_memory.promql

@@ -60,11 +60,17 @@ sum_over_time(
             # Join namespace label `label_appuio_io_organization` as `tenant_id`.
             on(cluster_id, namespace)
             group_left(tenant_id)
-            label_replace(
-              kube_namespace_labels{label_appuio_io_organization=~".+"},
-              "tenant_id",
-              "$1",
-              "label_appuio_io_organization", "(.*)"
+            (
+              bottomk(1,
+                min by (cluster_id, namespace, tenant_id) (
+                  label_replace(
+                    kube_namespace_labels{label_appuio_io_organization=~".+"},
+                    "tenant_id",
+                    "$1",
+                    "label_appuio_io_organization", "(.*)"
+                  )
+                )
+              ) by(cluster_id, namespace)
             ),
             # At least return 128MiB
             128 * 1024 * 1024

pkg/db/seeds/appuio_cloud_persistent_storage.promql

@@ -39,11 +39,17 @@ sum_over_time(
             # Join the namespace label to get the tenant
             on(cluster_id,namespace)
             group_left(tenant_id)
-            label_replace(
-              kube_namespace_labels{label_appuio_io_organization=~".+"},
-              "tenant_id",
-              "$1",
-              "label_appuio_io_organization", "(.*)"
+            (
+              bottomk(1,
+                min by (cluster_id, namespace, tenant_id) (
+                  label_replace(
+                    kube_namespace_labels{label_appuio_io_organization=~".+"},
+                    "tenant_id",
+                    "$1",
+                    "label_appuio_io_organization", "(.*)"
+                  )
+                )
+              ) by(cluster_id, namespace)
             )
           ),
           1024 * 1024 * 1024

pkg/db/seeds/promtest/appuio_cloud_loadbalancer.jsonnet

@@ -0,0 +1,91 @@
+local c = import 'common.libsonnet';
+
+local query = importstr '../appuio_cloud_loadbalancer.promql';
+
+local commonLabels = {
+  cluster_id: 'c-appuio-cloudscale-lpg-2',
+  tenant_id: 'c-appuio-cloudscale-lpg-2',
+};
+
+// One pvc, minimal (=1 byte) request
+// 10 samples
+local baseSeries = {
+  testprojectNamespaceOrgLabel: c.series('kube_namespace_labels', commonLabels {
+    namespace: 'testproject',
+    label_appuio_io_organization: 'cherry-pickers-inc',
+  }, '1x10'),
+
+  pvCapacity: c.series('kube_service_spec_type', commonLabels {
+    type: 'LoadBalancer',
+    namespace: 'testproject',
+  }, '1x10'),
+};
+
+local baseCalculatedLabels = {
+  category: 'c-appuio-cloudscale-lpg-2:testproject',
+  cluster_id: 'c-appuio-cloudscale-lpg-2',
+  namespace: 'testproject',
+  product: 'appuio_cloud_loadbalancer:c-appuio-cloudscale-lpg-2:cherry-pickers-inc:testproject',
+  tenant_id: 'cherry-pickers-inc',
+};
+
+{
+  tests: [
+    c.test('minimal PVC',
+           baseSeries,
+           query,
+           {
+             labels: c.formatLabels(baseCalculatedLabels),
+             value: 10,
+           }),
+
+    c.test('unrelated kube_namespace_labels changes do not throw errors - there is an overlap since series go stale only after a few missed scrapes',
+           baseSeries {
+             testprojectNamespaceOrgLabel+: {
+               values: '1x10 _x10 stale',
+             },
+             testprojectNamespaceOrgLabelUpdated: self.testprojectNamespaceOrgLabel {
+               _labels+:: {
+                 custom_appuio_io_myid: '672004be-a86b-44e0-b446-1255a1f8b340',
+               },
+               values: '_x5 1x15',
+             },
+           },
+           query,
+           {
+             labels: c.formatLabels(baseCalculatedLabels),
+             value: 10,
+           }),
+
+    c.test('organization changes do not throw many-to-many errors - there is an overlap since series go stale only after a few missed scrapes',
+           baseSeries {
+             testprojectNamespaceOrgLabel+: {
+               values: '1x7 _x10 stale',
+             },
+             testprojectNamespaceOrgLabelUpdated: self.testprojectNamespaceOrgLabel {
+               _labels+:: {
+                 label_appuio_io_organization: 'carrot-pickers-inc',
+               },
+               // We cheat here and use an impossible value.
+               // Since we use min() and bottomk() in the query this priotizes this series less than the other.
+               // It's ugly but it prevents flaky tests since otherwise one of the series gets picked randomly.
+               values: '_x2 2x15',
+             },
+           },
+           query,
+           [
+             {
+               labels: c.formatLabels(baseCalculatedLabels),
+               value: 8,
+             },
+             {
+               labels: c.formatLabels(baseCalculatedLabels {
+                 tenant_id: 'carrot-pickers-inc',
+                 product: 'appuio_cloud_loadbalancer:c-appuio-cloudscale-lpg-2:carrot-pickers-inc:testproject',
+               }),
+               // 1 service * two samples * 2 because of the cheat above.
+               value: 1 * 2 * 2,
+             },
+           ]),
+  ],
+}

pkg/db/seeds/promtest/appuio_cloud_memory.jsonnet

@@ -60,8 +60,25 @@ local baseCalculatedLabels = {
 local minMemoryRequestMib = 128;
 local cloudscaleFairUseRatio = 4294967296;
 
+local subQueryTests = [
+  c.test('sub CPU requests query sanity check',
+         baseSeries,
+         subCPUQuery,
+         {
+           labels: c.formatLabels(baseCalculatedLabels),
+           value: 0,
+         }),
+  c.test('sub memory requests query sanity check',
+         baseSeries,
+         subMemoryQuery,
+         {
+           labels: c.formatLabels(baseCalculatedLabels),
+           value: (minMemoryRequestMib - (1 / 1024 / 1024)) * 10,
+         }),
+];
+
 {
-  tests: [
+  tests: subQueryTests + [
     c.test('minimal pod',
            baseSeries,
            query,
@@ -193,19 +210,54 @@ local cloudscaleFairUseRatio = 4294967296;
              },
            ]),
 
-    c.test('sub CPU requests query sanity check',
-           baseSeries,
-           subCPUQuery,
-           {
-             labels: c.formatLabels(baseCalculatedLabels),
-             value: 0,
-           }),
-    c.test('sub memory requests query sanity check',
-           baseSeries,
-           subMemoryQuery,
+    c.test('unrelated kube_namespace_labels changes do not throw errors - there is an overlap since series go stale only after a few missed scrapes',
+           baseSeries {
+             testprojectNamespaceOrgLabel+: {
+               values: '1x10 _x10 stale',
+             },
+             testprojectNamespaceOrgLabelUpdated: self.testprojectNamespaceOrgLabel {
+               _labels+:: {
+                 custom_appuio_io_myid: '672004be-a86b-44e0-b446-1255a1f8b340',
+               },
+               values: '_x5 1x15',
+             },
+           },
+           query,
            {
              labels: c.formatLabels(baseCalculatedLabels),
-             value: (minMemoryRequestMib - (1 / 1024 / 1024)) * 10,
+             value: 128 * 10,
            }),
+
+    c.test('organization changes do not throw many-to-many errors - there is an overlap since series go stale only after a few missed scrapes',
+           baseSeries {
+             testprojectNamespaceOrgLabel+: {
+               values: '1x7 _x10 stale',
+             },
+             testprojectNamespaceOrgLabelUpdated: self.testprojectNamespaceOrgLabel {
+               _labels+:: {
+                 label_appuio_io_organization: 'carrot-pickers-inc',
+               },
+               // We cheat here and use an impossible value.
+               // Since we use min() and bottomk() in the query this priotizes this series less than the other.
+               // It's ugly but it prevents flaky tests since otherwise one of the series gets picked randomly.
+               // Does not influence the result. The result is flored to a minimum of 128MiB.
+               values: '_x2 2x15',
+             },
+           },
+           query,
+           [
+             {
+               labels: c.formatLabels(baseCalculatedLabels),
+               // Same as above it's 11*128 other queries don't have this problem
+               value: 128 * 9,
+             },
+             {
+               labels: c.formatLabels(baseCalculatedLabels {
+                 tenant_id: 'carrot-pickers-inc',
+                 product: 'appuio_cloud_memory:c-appuio-cloudscale-lpg-2:carrot-pickers-inc:testproject:flex',
+               }),
+               value: 128 * 2,
+             },
+           ]),
   ],
 }

pkg/db/seeds/promtest/appuio_cloud_persistent_storage.jsonnet

@@ -77,5 +77,54 @@ local baseCalculatedLabels = {
              labels: c.formatLabels(baseCalculatedLabels),
              value: 10,
            }),
+
+    c.test('unrelated kube_namespace_labels changes do not throw errors - there is an overlap since series go stale only after a few missed scrapes',
+           baseSeries {
+             testprojectNamespaceOrgLabel+: {
+               values: '1x10 _x10 stale',
+             },
+             testprojectNamespaceOrgLabelUpdated: self.testprojectNamespaceOrgLabel {
+               _labels+:: {
+                 custom_appuio_io_myid: '672004be-a86b-44e0-b446-1255a1f8b340',
+               },
+               values: '_x5 1x15',
+             },
+           },
+           query,
+           {
+             labels: c.formatLabels(baseCalculatedLabels),
+             value: 10,
+           }),
+
+    c.test('organization changes do not throw many-to-many errors - there is an overlap since series go stale only after a few missed scrapes',
+           baseSeries {
+             testprojectNamespaceOrgLabel+: {
+               values: '1x7 _x10 stale',
+             },
+             testprojectNamespaceOrgLabelUpdated: self.testprojectNamespaceOrgLabel {
+               _labels+:: {
+                 label_appuio_io_organization: 'carrot-pickers-inc',
+               },
+               // We cheat here and use an impossible value.
+               // Since we use min() and bottomk() in the query this priotizes this series less than the other.
+               // It's ugly but it prevents flaky tests since otherwise one of the series gets picked randomly.
+               // Does not influence the result. The result is flored to a minimum of 1GiB.
+               values: '_x2 2x15',
+             },
+           },
+           query,
+           [
+             {
+               labels: c.formatLabels(baseCalculatedLabels),
+               value: 8,
+             },
+             {
+               labels: c.formatLabels(baseCalculatedLabels {
+                 tenant_id: 'carrot-pickers-inc',
+                 product: 'appuio_cloud_persistent_storage:c-appuio-cloudscale-lpg-2:carrot-pickers-inc:testproject:ssd',
+               }),
+               value: 2,
+             },
+           ]),
   ],
 }