Skip to content

Commit

Permalink
Add support for explicit skips in all reporters
Browse files Browse the repository at this point in the history
  • Loading branch information
@psalz
psalz committed Dec 21, 2022
1 parent b323d67 commit 2841d77
Show file tree
Hide file tree
Showing 21 changed files with 2,559 additions and 2,261 deletions.
4 changes: 3 additions & 1 deletion src/catch2/reporters/catch_reporter_automake.cpp
View file
Expand Up @@ -17,7 +17,9 @@ namespace Catch {
void AutomakeReporter::testCaseEnded(TestCaseStats const& _testCaseStats) {
// Possible values to emit are PASS, XFAIL, SKIP, FAIL, XPASS and ERROR.
m_stream << ":test-result: ";
if (_testCaseStats.totals.assertions.allPassed()) {
if ( _testCaseStats.totals.testCases.skipped > 0 ) {
m_stream << "SKIP";
} else if (_testCaseStats.totals.assertions.allPassed()) {
m_stream << "PASS";
} else if (_testCaseStats.totals.assertions.allOk()) {
m_stream << "XFAIL";
Expand Down
8 changes: 6 additions & 2 deletions src/catch2/reporters/catch_reporter_compact.cpp
View file
Expand Up @@ -105,11 +105,15 @@ class AssertionPrinter {
printIssue("explicitly");
printRemainingMessages(Colour::None);
break;
case ResultWas::ExplicitSkip:
printResultType(Colour::Skip, "skipped"_sr);
printMessage();
printRemainingMessages();
break;
// These cases are here to prevent compiler warnings
case ResultWas::Unknown:
case ResultWas::FailureBit:
case ResultWas::Exception:
case ResultWas::ExplicitSkip:
printResultType(Colour::Error, "** internal error **");
break;
}
Expand Down Expand Up @@ -221,7 +225,7 @@ class AssertionPrinter {

// Drop out if result was successful and we're not printing those
if( !m_config->includeSuccessfulResults() && result.isOk() ) {
if( result.getResultType() != ResultWas::Warning )
if( result.getResultType() != ResultWas::Warning && result.getResultType() != ResultWas::ExplicitSkip )
return;
printInfoMessages = false;
}
Expand Down
15 changes: 9 additions & 6 deletions src/catch2/reporters/catch_reporter_junit.cpp
View file
Expand Up @@ -132,6 +132,7 @@ namespace Catch {
xml.writeAttribute( "name"_sr, stats.runInfo.name );
xml.writeAttribute( "errors"_sr, unexpectedExceptions );
xml.writeAttribute( "failures"_sr, stats.totals.assertions.failed-unexpectedExceptions );
xml.writeAttribute( "skipped"_sr, stats.totals.assertions.skipped );
xml.writeAttribute( "tests"_sr, stats.totals.assertions.total() );
xml.writeAttribute( "hostname"_sr, "tbd"_sr ); // !TBD
if( m_config->showDurations() == ShowDurations::Never )
Expand Down Expand Up @@ -244,7 +245,8 @@ namespace Catch {

void JunitReporter::writeAssertion( AssertionStats const& stats ) {
AssertionResult const& result = stats.assertionResult;
if( !result.isOk() ) {
if ( !result.isOk() ||
result.getResultType() == ResultWas::ExplicitSkip ) {
std::string elementName;
switch( result.getResultType() ) {
case ResultWas::ThrewException:
Expand All @@ -256,15 +258,16 @@ namespace Catch {
case ResultWas::DidntThrowException:
elementName = "failure";
break;

case ResultWas::ExplicitSkip:
elementName = "skipped";
break;
// We should never see these here:
case ResultWas::Info:
case ResultWas::Warning:
case ResultWas::Ok:
case ResultWas::Unknown:
case ResultWas::FailureBit:
case ResultWas::Exception:
case ResultWas::ExplicitSkip:
elementName = "internalError";
break;
}
Expand All @@ -275,7 +278,9 @@ namespace Catch {
xml.writeAttribute( "type"_sr, result.getTestMacroName() );

ReusableStringStream rss;
if (stats.totals.assertions.total() > 0) {
if ( result.getResultType() == ResultWas::ExplicitSkip ) {
rss << "SKIPPED\n";
} else {
rss << "FAILED" << ":\n";
if (result.hasExpression()) {
rss << " ";
Expand All @@ -286,8 +291,6 @@ namespace Catch {
rss << "with expansion:\n";
rss << TextFlow::Column(result.getExpandedExpression()).indent(2) << '\n';
}
} else {
rss << '\n';
}

if( !result.getMessage().empty() )
Expand Down
16 changes: 8 additions & 8 deletions src/catch2/reporters/catch_reporter_sonarqube.cpp
View file
Expand Up @@ -97,7 +97,8 @@ namespace Catch {

void SonarQubeReporter::writeAssertion(AssertionStats const& stats, bool okToFail) {
AssertionResult const& result = stats.assertionResult;
if (!result.isOk()) {
if ( !result.isOk() ||
result.getResultType() == ResultWas::ExplicitSkip ) {
std::string elementName;
if (okToFail) {
elementName = "skipped";
Expand All @@ -108,23 +109,20 @@ namespace Catch {
elementName = "error";
break;
case ResultWas::ExplicitFailure:
elementName = "failure";
break;
case ResultWas::ExpressionFailed:
elementName = "failure";
break;
case ResultWas::DidntThrowException:
elementName = "failure";
break;

case ResultWas::ExplicitSkip:
elementName = "skipped";
break;
// We should never see these here:
case ResultWas::Info:
case ResultWas::Warning:
case ResultWas::Ok:
case ResultWas::Unknown:
case ResultWas::FailureBit:
case ResultWas::Exception:
case ResultWas::ExplicitSkip:
elementName = "internalError";
break;
}
Expand All @@ -137,7 +135,9 @@ namespace Catch {
xml.writeAttribute("message"_sr, messageRss.str());

ReusableStringStream textRss;
if (stats.totals.assertions.total() > 0) {
if ( result.getResultType() == ResultWas::ExplicitSkip ) {
textRss << "SKIPPED\n";
} else {
textRss << "FAILED:\n";
if (result.hasExpression()) {
textRss << '\t' << result.getExpressionInMacro() << '\n';
Expand Down
7 changes: 6 additions & 1 deletion src/catch2/reporters/catch_reporter_tap.cpp
View file
Expand Up @@ -100,11 +100,16 @@ namespace Catch {
printIssue("explicitly"_sr);
printRemainingMessages(Colour::None);
break;
case ResultWas::ExplicitSkip:
printResultType(tapPassedString);
printIssue(" # SKIP"_sr);
printMessage();
printRemainingMessages();
break;
// These cases are here to prevent compiler warnings
case ResultWas::Unknown:
case ResultWas::FailureBit:
case ResultWas::Exception:
case ResultWas::ExplicitSkip:
printResultType("** internal error **"_sr);
break;
}
Expand Down
23 changes: 12 additions & 11 deletions src/catch2/reporters/catch_reporter_teamcity.cpp
View file
Expand Up @@ -59,7 +59,8 @@ namespace Catch {

void TeamCityReporter::assertionEnded(AssertionStats const& assertionStats) {
AssertionResult const& result = assertionStats.assertionResult;
if (!result.isOk()) {
if ( !result.isOk() ||
result.getResultType() == ResultWas::ExplicitSkip ) {

ReusableStringStream msg;
if (!m_headerPrintedForThisSection)
Expand All @@ -84,12 +85,14 @@ namespace Catch {
case ResultWas::ExplicitFailure:
msg << "explicit failure";
break;
case ResultWas::ExplicitSkip:
msg << "explicit skip";
break;

// We shouldn't get here because of the isOk() test
case ResultWas::Ok:
case ResultWas::Info:
case ResultWas::Warning:
case ResultWas::ExplicitSkip:
CATCH_ERROR("Internal error in TeamCity reporter");
// These cases are here to prevent compiler warnings
case ResultWas::Unknown:
Expand All @@ -112,18 +115,16 @@ namespace Catch {
" " << result.getExpandedExpression() << '\n';
}

if (currentTestCaseInfo->okToFail()) {
if ( result.getResultType() == ResultWas::ExplicitSkip ) {
m_stream << "##teamcity[testIgnored";
} else if ( currentTestCaseInfo->okToFail() ) {
msg << "- failure ignore as test marked as 'ok to fail'\n";
m_stream << "##teamcity[testIgnored"
<< " name='" << escape(currentTestCaseInfo->name) << '\''
<< " message='" << escape(msg.str()) << '\''
<< "]\n";
m_stream << "##teamcity[testIgnored";
} else {
m_stream << "##teamcity[testFailed"
<< " name='" << escape(currentTestCaseInfo->name) << '\''
<< " message='" << escape(msg.str()) << '\''
<< "]\n";
m_stream << "##teamcity[testFailed";
}
m_stream << " name='" << escape( currentTestCaseInfo->name ) << '\''
<< " message='" << escape( msg.str() ) << '\'' << "]\n";
}
m_stream.flush();
}
Expand Down
19 changes: 15 additions & 4 deletions src/catch2/reporters/catch_reporter_xml.cpp
View file
Expand Up @@ -108,9 +108,10 @@ namespace Catch {
}

// Drop out if result was successful but we're not printing them.
if( !includeResults && result.getResultType() != ResultWas::Warning )
if ( !includeResults && result.getResultType() != ResultWas::Warning &&
result.getResultType() != ResultWas::ExplicitSkip ) {
return;

}

// Print the expression if there is one.
if( result.hasExpression() ) {
Expand Down Expand Up @@ -153,6 +154,12 @@ namespace Catch {
m_xml.writeText( result.getMessage() );
m_xml.endElement();
break;
case ResultWas::ExplicitSkip:
m_xml.startElement( "Skip" );
writeSourceInfo( result.getSourceInfo() );
m_xml.writeText( result.getMessage() );
m_xml.endElement();
break;
default:
break;
}
Expand All @@ -168,6 +175,7 @@ namespace Catch {
e.writeAttribute( "successes"_sr, sectionStats.assertions.passed );
e.writeAttribute( "failures"_sr, sectionStats.assertions.failed );
e.writeAttribute( "expectedFailures"_sr, sectionStats.assertions.failedButOk );
e.writeAttribute( "skipped"_sr, sectionStats.assertions.skipped > 0 );

if ( m_config->showDurations() == ShowDurations::Always )
e.writeAttribute( "durationInSeconds"_sr, sectionStats.durationInSeconds );
Expand All @@ -180,6 +188,7 @@ namespace Catch {
StreamingReporterBase::testCaseEnded( testCaseStats );
XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResult" );
e.writeAttribute( "success"_sr, testCaseStats.totals.assertions.allOk() );
e.writeAttribute( "skips"_sr, testCaseStats.totals.assertions.skipped );

if ( m_config->showDurations() == ShowDurations::Always )
e.writeAttribute( "durationInSeconds"_sr, m_testCaseTimer.getElapsedSeconds() );
Expand All @@ -197,11 +206,13 @@ namespace Catch {
m_xml.scopedElement( "OverallResults" )
.writeAttribute( "successes"_sr, testRunStats.totals.assertions.passed )
.writeAttribute( "failures"_sr, testRunStats.totals.assertions.failed )
.writeAttribute( "expectedFailures"_sr, testRunStats.totals.assertions.failedButOk );
.writeAttribute( "expectedFailures"_sr, testRunStats.totals.assertions.failedButOk )
.writeAttribute( "skips"_sr, testRunStats.totals.assertions.skipped );
m_xml.scopedElement( "OverallResultsCases")
.writeAttribute( "successes"_sr, testRunStats.totals.testCases.passed )
.writeAttribute( "failures"_sr, testRunStats.totals.testCases.failed )
.writeAttribute( "expectedFailures"_sr, testRunStats.totals.testCases.failedButOk );
.writeAttribute( "expectedFailures"_sr, testRunStats.totals.testCases.failedButOk )
.writeAttribute( "skips"_sr, testRunStats.totals.testCases.skipped );
m_xml.endElement();
}

Expand Down
10 changes: 5 additions & 5 deletions tests/SelfTest/Baselines/automake.sw.approved.txt
View file
Expand Up @@ -297,7 +297,7 @@ Message from section two
:test-result: PASS X/level/1/b
:test-result: PASS XmlEncode
:test-result: PASS XmlWriter writes boolean attributes as true/false
:test-result: XFAIL a succeeding test can still be skipped
:test-result: SKIP a succeeding test can still be skipped
:test-result: PASS analyse no analysis
:test-result: PASS array<int, N> -> toString
:test-result: PASS benchmark function call
Expand All @@ -310,7 +310,7 @@ Message from section two
:test-result: PASS comparisons between const int variables
:test-result: PASS comparisons between int variables
:test-result: PASS convertToBits
:test-result: XFAIL dynamic skipping works with generators
:test-result: SKIP dynamic skipping works with generators
:test-result: PASS empty tags are not allowed
:test-result: PASS erfc_inv
:test-result: PASS estimate_clock_resolution
Expand Down Expand Up @@ -361,11 +361,11 @@ b1!
:test-result: PASS run_for_at_least, chronometer
:test-result: PASS run_for_at_least, int
:test-result: FAIL second tag
:test-result: XFAIL sections can be skipped dynamically at runtime
:test-result: SKIP sections can be skipped dynamically at runtime
:test-result: FAIL send a single char to INFO
:test-result: FAIL sends information to INFO
:test-result: PASS shortened hide tags are split apart
:test-result: XFAIL skipped tests can optionally provide a reason
:test-result: SKIP skipped tests can optionally provide a reason
:test-result: PASS splitString
:test-result: FAIL stacks unscoped info in loops
:test-result: PASS startsWith
Expand All @@ -387,7 +387,7 @@ b1!
:test-result: PASS strlen3
:test-result: PASS tables
:test-result: PASS tags with dots in later positions are not parsed as hidden
:test-result: XFAIL tests can be skipped dynamically at runtime
:test-result: SKIP tests can be skipped dynamically at runtime
:test-result: FAIL thrown std::strings are translated
:test-result: PASS toString on const wchar_t const pointer returns the string contents
:test-result: PASS toString on const wchar_t pointer returns the string contents
Expand Down
10 changes: 5 additions & 5 deletions tests/SelfTest/Baselines/automake.sw.multi.approved.txt
View file
Expand Up @@ -290,7 +290,7 @@
:test-result: PASS X/level/1/b
:test-result: PASS XmlEncode
:test-result: PASS XmlWriter writes boolean attributes as true/false
:test-result: XFAIL a succeeding test can still be skipped
:test-result: SKIP a succeeding test can still be skipped
:test-result: PASS analyse no analysis
:test-result: PASS array<int, N> -> toString
:test-result: PASS benchmark function call
Expand All @@ -303,7 +303,7 @@
:test-result: PASS comparisons between const int variables
:test-result: PASS comparisons between int variables
:test-result: PASS convertToBits
:test-result: XFAIL dynamic skipping works with generators
:test-result: SKIP dynamic skipping works with generators
:test-result: PASS empty tags are not allowed
:test-result: PASS erfc_inv
:test-result: PASS estimate_clock_resolution
Expand Down Expand Up @@ -350,11 +350,11 @@
:test-result: PASS run_for_at_least, chronometer
:test-result: PASS run_for_at_least, int
:test-result: FAIL second tag
:test-result: XFAIL sections can be skipped dynamically at runtime
:test-result: SKIP sections can be skipped dynamically at runtime
:test-result: FAIL send a single char to INFO
:test-result: FAIL sends information to INFO
:test-result: PASS shortened hide tags are split apart
:test-result: XFAIL skipped tests can optionally provide a reason
:test-result: SKIP skipped tests can optionally provide a reason
:test-result: PASS splitString
:test-result: FAIL stacks unscoped info in loops
:test-result: PASS startsWith
Expand All @@ -376,7 +376,7 @@
:test-result: PASS strlen3
:test-result: PASS tables
:test-result: PASS tags with dots in later positions are not parsed as hidden
:test-result: XFAIL tests can be skipped dynamically at runtime
:test-result: SKIP tests can be skipped dynamically at runtime
:test-result: FAIL thrown std::strings are translated
:test-result: PASS toString on const wchar_t const pointer returns the string contents
:test-result: PASS toString on const wchar_t pointer returns the string contents
Expand Down

0 comments on commit 2841d77

Please sign in to comment.