JSON output format doesn't include group thresholds

[lightindigo]

Brief summary

I've got scenarios that may include several actions and I have the "target_group" which includes actions that I'd like to check thresholds on.

If I specify thresholds for this group specifically, I can see no thresholds info while using --out json=metrics.json

k6 version

0.48,0.49

OS

macOs 14.13.1

Docker version and image (if applicable)

No response

Steps to reproduce the problem

Copy folowing JS script into sample-test.js

import http from 'k6/http';
import {group} from "k6";

export let options = {
    vus: 10,
    duration: '5s',
    thresholds: {
        'checks{group:::target_group}': ['rate>0.9999'],
        'http_req_duration{group:::target_group}': ['p(95)<100'],
        'http_req_failed{group:::target_group}': ['rate<0.01']
    }
};

export default () => {
    group('target_group', function () {
        http.get('https://test.k6.io');
    });
};

run

k6 sample-test.js  --out json=metrics.json

and check

Expected behaviour

What I get without specifying group in thresholds

{"type":"Metric","data":{"name":"data_received","type":"counter","contains":"data","thresholds":[],"submetrics":null},"metric":"data_received"}
{"type":"Metric","data":{"name":"data_sent","type":"counter","contains":"data","thresholds":[],"submetrics":null},"metric":"data_sent"}
{"type":"Metric","data":{"name":"group_duration","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"group_duration"}
{"type":"Metric","data":{"name":"http_req_blocked","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_blocked"}
{"type":"Metric","data":{"name":"http_req_connecting","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_connecting"}
{"type":"Metric","data":{"name":"http_req_duration","type":"trend","contains":"time","thresholds":["p(95)<100"],"submetrics":[{"name":"http_req_duration{expected_response:true}","suffix":"expected_response:true","tags":{"expected_response":"true"}}]},"metric":"http_req_duration"}
{"type":"Metric","data":{"name":"http_req_failed","type":"rate","contains":"default","thresholds":["rate<0.01"],"submetrics":null},"metric":"http_req_failed"}
{"type":"Metric","data":{"name":"http_req_receiving","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_receiving"}
{"type":"Metric","data":{"name":"http_req_sending","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_sending"}
{"type":"Metric","data":{"name":"http_req_tls_handshaking","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_tls_handshaking"}
{"type":"Metric","data":{"name":"http_req_waiting","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_waiting"}
{"type":"Metric","data":{"name":"http_reqs","type":"counter","contains":"default","thresholds":[],"submetrics":null},"metric":"http_reqs"}
{"type":"Metric","data":{"name":"iteration_duration","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"iteration_duration"}
{"type":"Metric","data":{"name":"iterations","type":"counter","contains":"default","thresholds":[],"submetrics":null},"metric":"iterations"}
{"type":"Metric","data":{"name":"vus","type":"gauge","contains":"default","thresholds":[],"submetrics":null},"metric":"vus"}
{"type":"Metric","data":{"name":"vus_max","type":"gauge","contains":"default","thresholds":[],"submetrics":null},"metric":"vus_max"}

Actual behaviour

{"type":"Metric","data":{"name":"data_received","type":"counter","contains":"data","thresholds":[],"submetrics":null},"metric":"data_received"}
{"type":"Metric","data":{"name":"data_sent","type":"counter","contains":"data","thresholds":[],"submetrics":null},"metric":"data_sent"}
{"type":"Metric","data":{"name":"group_duration","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"group_duration"}
{"type":"Metric","data":{"name":"http_req_blocked","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_blocked"}
{"type":"Metric","data":{"name":"http_req_connecting","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_connecting"}
{"type":"Metric","data":{"name":"http_req_duration","type":"trend","contains":"time","thresholds":[],"submetrics":[{"name":"http_req_duration{group:::target_group}","suffix":"group:::target_group","tags":{"group":"::target_group"}},{"name":"http_req_duration{expected_response:true}","suffix":"expected_response:true","tags":{"expected_response":"true"}}]},"metric":"http_req_duration"}
{"type":"Metric","data":{"name":"http_req_failed","type":"rate","contains":"default","thresholds":[],"submetrics":[{"name":"http_req_failed{group:::target_group}","suffix":"group:::target_group","tags":{"group":"::target_group"}}]},"metric":"http_req_failed"}
{"type":"Metric","data":{"name":"http_req_receiving","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_receiving"}
{"type":"Metric","data":{"name":"http_req_sending","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_sending"}
{"type":"Metric","data":{"name":"http_req_tls_handshaking","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_tls_handshaking"}
{"type":"Metric","data":{"name":"http_req_waiting","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"http_req_waiting"}
{"type":"Metric","data":{"name":"http_reqs","type":"counter","contains":"default","thresholds":[],"submetrics":null},"metric":"http_reqs"}
{"type":"Metric","data":{"name":"iteration_duration","type":"trend","contains":"time","thresholds":[],"submetrics":null},"metric":"iteration_duration"}
{"type":"Metric","data":{"name":"iterations","type":"counter","contains":"default","thresholds":[],"submetrics":null},"metric":"iterations"}
{"type":"Metric","data":{"name":"vus","type":"gauge","contains":"default","thresholds":[],"submetrics":null},"metric":"vus"}
{"type":"Metric","data":{"name":"vus_max","type":"gauge","contains":"default","thresholds":[],"submetrics":null},"metric":"vus_max"}

[oleiade]

Hi @lightindigo 👋🏻

Thanks for bringing this to our attention 🙇‍♂️

My understanding of the issue you are experiencing is that when defining thresholds over a specific group, those are evaluated as expected, but their value is not reflected in the .data.thresholds field of the JSON output, which is what you would expect. Did I understand it correctly?

If that's the case, I was able to reproduce this indeed.

with a script defining thresholds without filtering on a group

import http from 'k6/http';
import {group} from "k6";

export let options = {
    vus: 10,
    duration: '5s',
    thresholds: {
        'http_req_duration': ['p(95)<100'],
        'http_req_failed': ['rate<0.01'],
    }
};

export default () => {
    group('target_group', function () {
        http.get('https://test.k6.io');
    });
};

The JSON output relevant entries have their .data.thresholds file set

 > cat -p metrics_no_group.json | jq 'select((.data.thresholds != null) and (.data.thresholds | length > 0))'
{
  "type": "Metric",
  "data": {
    "name": "http_req_duration",
    "type": "trend",
    "contains": "time",
    "thresholds": [
      "p(95)<100"
    ],
    "submetrics": [
      {
        "name": "http_req_duration{expected_response:true}",
        "suffix": "expected_response:true",
        "tags": {
          "expected_response": "true"
        }
      }
    ]
  },
  "metric": "http_req_duration"
}
{
  "type": "Metric",
  "data": {
    "name": "http_req_failed",
    "type": "rate",
    "contains": "default",
    "thresholds": [
      "rate<0.01"
    ],
    "submetrics": null
  },
  "metric": "http_req_failed"
}

With a script defining thresholds filtering on a specific group

import http from 'k6/http';
import {group} from "k6";

export let options = {
    vus: 10,
    duration: '5s',
    thresholds: {
        'checks{group:::target_group}': ['rate>0.9999'],
        'http_req_duration{group:::target_group}': ['p(95)<100'],
        'http_req_failed{group:::target_group}': ['rate<0.01']
    }
};

export default () => {
    group('target_group', function () {
        http.get('https://test.k6.io');
    });
};

The JSON output relevant entries have their .data.thresholds field left empty

> cat -p metrics_with.json | jq 'select((.data.thresholds != null) and (.data.thresholds | length > 0))'

Regardless, I haven't touched thresholds in a rather long time, and I'm not certain whether this is a bug, or an undocumented behavior yet, I'll look into it.

[lightindigo]

Yeah, that's right. The issue is in JSON output only, the thresholds are definitely evaluated as expected