feat: Add JUnit output format for test results

[oguzhand95]

References

https://github.com/gotestyourself/gotestsum/tree/main/internal/junitxml
https://github.com/testmoapp/junitxml

---

Example Output (-ojunit)

<testsuites errors="1" failures="2" skipped="4" tests="15">
  <testsuite description="Tests for verifying the company resource policy" name="CompanyTestSuite" file="company_test.yaml" errors="0" failures="2" skipped="4" tests="12">
    <properties></properties>
    <testcase file="company_test.yaml" classname="user.company.create" name="Company Test 1">
      <failure type="RESULT_FAILED">
        <actual>EFFECT_ALLOW</actual>
        <expected>EFFECT_DENY</expected>
      </failure>
    </testcase>
    <testcase file="company_test.yaml" classname="user.company.read" name="Company Test 1">
      <failure type="RESULT_FAILED">
        <actual>EFFECT_ALLOW</actual>
        <expected>EFFECT_DENY</expected>
      </failure>
    </testcase>
    <testcase file="company_test.yaml" classname="admin.company.create" name="Company Test 3">
      <skipped message="This test was skipped"></skipped>
    </testcase>
    <testcase file="company_test.yaml" classname="admin.company.read" name="Company Test 3">
      <skipped message="This test was skipped"></skipped>
    </testcase>
    <testcase file="company_test.yaml" classname="user.company.create" name="Company Test 3">
      <skipped message="This test was skipped"></skipped>
    </testcase>
    <testcase file="company_test.yaml" classname="user.company.read" name="Company Test 3">
      <skipped message="This test was skipped"></skipped>
    </testcase>
  </testsuite>
  <testsuite description="Tests for verifying the contact resource policy" name="ContactTestSuite" file="contact_test.yaml" errors="0" failures="0" skipped="0" tests="3">
    <properties></properties>
  </testsuite>
  <testsuite description="Tests for verifying the user resource policy" name="UserTestSuite" file="user_test.yaml" errors="1" failures="0" skipped="0" tests="0">
    <error type="RESULT_ERRORED">Invalid test suite: another test named User Test 1 already exists</error>
    <properties></properties>
  </testsuite>
</testsuites>

---

Example Output (--verbose -ojunit)

<testsuites errors="1" failures="2" skipped="4" tests="15">
  <testsuite description="Tests for verifying the company resource policy" name="CompanyTestSuite" file="company_test.yaml" errors="0" failures="2" skipped="4" tests="12">
    <properties>
      <property name="step[Company Test 1 - user.company.create]">
        <traces>
          <trace id="1">
            <condition>KIND_CONDITION</condition>
            <action>create</action>
            <policy>cerbos.resource.company.vdefault</policy>
            <rule>rule-001</rule>
            <result>true</result>
            <activated>true</activated>
          </trace>
          <trace id="2">
            <action>create</action>
            <policy>cerbos.resource.company.vdefault</policy>
            <rule>rule-001</rule>
            <effect>EFFECT_ALLOW</effect>
            <activated>true</activated>
          </trace>
          <trace id="3">
            <policy>cerbos.resource.company.vdefault</policy>
            <rule>rule-002</rule>
            <message>No matching roles or derived roles</message>
            <skipped>true</skipped>
          </trace>
        </traces>
      </property>
      <property name="step[Company Test 1 - user.company.read]">
        <traces>
          <trace id="1">
            <condition>KIND_CONDITION</condition>
            <action>read</action>
            <policy>cerbos.resource.company.vdefault</policy>
            <rule>rule-001</rule>
            <result>true</result>
            <activated>true</activated>
          </trace>
          <trace id="2">
            <action>read</action>
            <policy>cerbos.resource.company.vdefault</policy>
            <rule>rule-001</rule>
            <effect>EFFECT_ALLOW</effect>
            <activated>true</activated>
          </trace>
          <trace id="3">
            <policy>cerbos.resource.company.vdefault</policy>
            <rule>rule-002</rule>
            <message>No matching roles or derived roles</message>
            <skipped>true</skipped>
          </trace>
        </traces>
      </property>
    </properties>
    <testcase file="company_test.yaml" classname="admin.company.create" name="Company Test 1"></testcase>
    <testcase file="company_test.yaml" classname="admin.company.read" name="Company Test 1"></testcase>
    <testcase file="company_test.yaml" classname="user.company.create" name="Company Test 1">
      <failure type="RESULT_FAILED">
        <actual>EFFECT_ALLOW</actual>
        <expected>EFFECT_DENY</expected>
      </failure>
    </testcase>
    <testcase file="company_test.yaml" classname="user.company.read" name="Company Test 1">
      <failure type="RESULT_FAILED">
        <actual>EFFECT_ALLOW</actual>
        <expected>EFFECT_DENY</expected>
      </failure>
    </testcase>
    <testcase file="company_test.yaml" classname="admin.company.create" name="Company Test 2"></testcase>
    <testcase file="company_test.yaml" classname="admin.company.read" name="Company Test 2"></testcase>
    <testcase file="company_test.yaml" classname="user.company.create" name="Company Test 2"></testcase>
    <testcase file="company_test.yaml" classname="user.company.read" name="Company Test 2"></testcase>
    <testcase file="company_test.yaml" classname="admin.company.create" name="Company Test 3">
      <skipped message="This test was skipped"></skipped>
    </testcase>
    <testcase file="company_test.yaml" classname="admin.company.read" name="Company Test 3">
      <skipped message="This test was skipped"></skipped>
    </testcase>
    <testcase file="company_test.yaml" classname="user.company.create" name="Company Test 3">
      <skipped message="This test was skipped"></skipped>
    </testcase>
    <testcase file="company_test.yaml" classname="user.company.read" name="Company Test 3">
      <skipped message="This test was skipped"></skipped>
    </testcase>
  </testsuite>
  <testsuite description="Tests for verifying the contact resource policy" name="ContactTestSuite" file="contact_test.yaml" errors="0" failures="0" skipped="0" tests="3">
    <properties></properties>
    <testcase file="contact_test.yaml" classname="admin.contact.create" name="Contact Test 1"></testcase>
    <testcase file="contact_test.yaml" classname="admin.contact.create" name="Contact Test 2"></testcase>
    <testcase file="contact_test.yaml" classname="admin.contact.create" name="Contact Test 3"></testcase>
  </testsuite>
  <testsuite description="Tests for verifying the user resource policy" name="UserTestSuite" file="user_test.yaml" errors="1" failures="0" skipped="0" tests="0">
    <error type="RESULT_ERRORED">Invalid test suite: another test named User Test 1 already exists</error>
    <properties></properties>
  </testsuite>
</testsuites>

[codecov]

Codecov Report

Merging #1508 (2c42c61) into main (f40a8a8) will increase coverage by 0.38%.
The diff coverage is 81.90%.

Additional details and impacted files

@@            Coverage Diff             @@
##             main    #1508      +/-   ##
==========================================
+ Coverage   53.68%   54.06%   +0.38%     
==========================================
  Files         121      122       +1     
  Lines       13472    13571      +99     
==========================================
+ Hits         7232     7337     +105     
+ Misses       5618     5609       -9     
- Partials      622      625       +3     

Impacted Files	Coverage Δ
internal/policy/policy.go	35.11% <ø> (ø)
internal/verify/junit/junit.go	80.41% <80.41%> (ø)
internal/verify/test_fixture.go	86.28% <100.00%> (+0.04%)	⬆️
internal/verify/verify.go	83.78% <100.00%> (+0.14%)	⬆️

... and 4 files with indirect coverage changes

[charithe]

Nice!

Test suites have a description field so I think that should go into the XML description attribute. There's a filename attribute for storing the file name.

- <testsuite name="company_test.yaml" description="CompanyTestSuite" errors="0" failures="0" skipped="4" tests="12">
+ <testsuite name="CompanyTestSuite" description="My company test" file="company_test.yaml"  errors="0" failures="0" skipped="4" tests="12">

Even testcases have a file attribute so it'd be good to populate that as well.

I think we can use steps to render the execution trace from --verbose.

[charithe]

Nice work! There are a couple of small issues to fix:

• If the output format is set to junit, compile or lint errors don't get displayed.
• I realise now that the traces require more work because we have to essentially define the schema for how to render them. Therefore, let's remove that feature from this PR. Being able to render the test failures and successes without traces is already a big improvement. We can add traces later in a different PR.

Just FYI, the reason why traces require more work is because of the following:

• The traces need to go inside the corresponding test case rather than the test suite.
• I think that the trace output is supposed to look (roughly) like the following. (We need to work out the correct schema first.)

<property="step[cerbos.resource.company.vdefault - rule-001]" value="activated">
   <trace>
      <condition><![CDATA[R.id == "test"]]></condition>
      <action>read</action>
      <policy>cerbos.resource.company.vdefault</policy>
      <rule>rule-001</rule>
      <result>true</result>
      <outcome>activated</outcome>
   </trace>
</property>
<property="step[cerbos.resource.company.vdefault - rule-002]" value="skipped">
   <trace>
      <action>read</action>
      <policy>cerbos.resource.company.vdefault</policy>
      <rule>rule-002</rule>
      <outcome>skipped</outcome>
   </trace>
</property>

[charithe]

Let's make sure that the test output flag value defaults to the value of the output flag if the former is not explicitly set.

Regarding the problem of distinguishing why the command failed (lint, compile, or test) I think we can return different exit codes when the program terminates. You can override the default exit handler using kong.Exit. So, the exit codes could be the following:

• 3 - Lint failure
• 4 - Compile failure
• 5 - Test failure

Finally, let's update the docs pages to include the new test-output flag and describe how it works.

[charithe]

👍🏽