diff --git a/src/catch2/catch_test_macros.hpp b/src/catch2/catch_test_macros.hpp index 792fdce96e..89340a491e 100644 --- a/src/catch2/catch_test_macros.hpp +++ b/src/catch2/catch_test_macros.hpp @@ -49,6 +49,7 @@ #define CATCH_FAIL( ... ) INTERNAL_CATCH_MSG( "CATCH_FAIL", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, __VA_ARGS__ ) #define CATCH_FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( "CATCH_FAIL_CHECK", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) #define CATCH_SUCCEED( ... ) INTERNAL_CATCH_MSG( "CATCH_SUCCEED", Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) + #define CATCH_SKIP( ... ) INTERNAL_CATCH_MSG( "SKIP", Catch::ResultWas::ExplicitSkip, Catch::ResultDisposition::Normal, __VA_ARGS__ ) #if !defined(CATCH_CONFIG_RUNTIME_STATIC_REQUIRE) @@ -102,6 +103,7 @@ #define CATCH_FAIL( ... ) (void)(0) #define CATCH_FAIL_CHECK( ... ) (void)(0) #define CATCH_SUCCEED( ... ) (void)(0) + #define CATCH_SKIP( ... ) (void)(0) #define CATCH_STATIC_REQUIRE( ... ) (void)(0) #define CATCH_STATIC_REQUIRE_FALSE( ... ) (void)(0) @@ -146,6 +148,7 @@ #define FAIL( ... ) INTERNAL_CATCH_MSG( "FAIL", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, __VA_ARGS__ ) #define FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( "FAIL_CHECK", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) #define SUCCEED( ... ) INTERNAL_CATCH_MSG( "SUCCEED", Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) + #define SKIP( ... ) INTERNAL_CATCH_MSG( "SKIP", Catch::ResultWas::ExplicitSkip, Catch::ResultDisposition::Normal, __VA_ARGS__ ) #if !defined(CATCH_CONFIG_RUNTIME_STATIC_REQUIRE) @@ -198,6 +201,7 @@ #define FAIL( ... ) (void)(0) #define FAIL_CHECK( ... ) (void)(0) #define SUCCEED( ... ) (void)(0) + #define SKIP( ... ) (void)(0) #define STATIC_REQUIRE( ... ) (void)(0) #define STATIC_REQUIRE_FALSE( ... ) (void)(0) diff --git a/src/catch2/catch_totals.cpp b/src/catch2/catch_totals.cpp index 14a23869df..e139aa124c 100644 --- a/src/catch2/catch_totals.cpp +++ b/src/catch2/catch_totals.cpp @@ -14,6 +14,7 @@ namespace Catch { diff.passed = passed - other.passed; diff.failed = failed - other.failed; diff.failedButOk = failedButOk - other.failedButOk; + diff.skipped = skipped - other.skipped; return diff; } @@ -21,14 +22,15 @@ namespace Catch { passed += other.passed; failed += other.failed; failedButOk += other.failedButOk; + skipped += other.skipped; return *this; } std::uint64_t Counts::total() const { - return passed + failed + failedButOk; + return passed + failed + failedButOk + skipped; } bool Counts::allPassed() const { - return failed == 0 && failedButOk == 0; + return failed == 0 && failedButOk == 0 && skipped == 0; } bool Counts::allOk() const { return failed == 0; @@ -53,6 +55,8 @@ namespace Catch { ++diff.testCases.failed; else if( diff.assertions.failedButOk > 0 ) ++diff.testCases.failedButOk; + else if ( diff.assertions.skipped > 0 ) + ++ diff.testCases.skipped; else ++diff.testCases.passed; return diff; diff --git a/src/catch2/catch_totals.hpp b/src/catch2/catch_totals.hpp index 7ab6a1b611..b43a6116f5 100644 --- a/src/catch2/catch_totals.hpp +++ b/src/catch2/catch_totals.hpp @@ -24,6 +24,7 @@ namespace Catch { std::uint64_t passed = 0; std::uint64_t failed = 0; std::uint64_t failedButOk = 0; + std::uint64_t skipped = 0; }; struct Totals { diff --git a/src/catch2/internal/catch_assertion_handler.cpp b/src/catch2/internal/catch_assertion_handler.cpp index 828aa7cdff..d0080d0845 100644 --- a/src/catch2/internal/catch_assertion_handler.cpp +++ b/src/catch2/internal/catch_assertion_handler.cpp @@ -52,6 +52,13 @@ namespace Catch { throw Catch::TestFailureException(); #else CATCH_ERROR( "Test failure requires aborting test!" ); +#endif + } + if ( m_reaction.shouldSkip ) { +#if !defined( CATCH_CONFIG_DISABLE_EXCEPTIONS ) + throw Catch::TestSkipException(); +#else + CATCH_ERROR( "Explicitly skipping tests during runtime requires exceptions" ); #endif } } diff --git a/src/catch2/internal/catch_assertion_handler.hpp b/src/catch2/internal/catch_assertion_handler.hpp index b7c7613e31..bbbcf29c22 100644 --- a/src/catch2/internal/catch_assertion_handler.hpp +++ b/src/catch2/internal/catch_assertion_handler.hpp @@ -22,6 +22,7 @@ namespace Catch { struct AssertionReaction { bool shouldDebugBreak = false; bool shouldThrow = false; + bool shouldSkip = false; }; class AssertionHandler { diff --git a/src/catch2/internal/catch_console_colour.hpp b/src/catch2/internal/catch_console_colour.hpp index 7b102013c8..e8bc67634c 100644 --- a/src/catch2/internal/catch_console_colour.hpp +++ b/src/catch2/internal/catch_console_colour.hpp @@ -41,6 +41,7 @@ namespace Catch { Error = BrightRed, Success = Green, + Skip = Blue, OriginalExpression = Cyan, ReconstructedExpression = BrightYellow, diff --git a/src/catch2/internal/catch_exception_translator_registry.cpp b/src/catch2/internal/catch_exception_translator_registry.cpp index acddea6ec9..8bc8c54efa 100644 --- a/src/catch2/internal/catch_exception_translator_registry.cpp +++ b/src/catch2/internal/catch_exception_translator_registry.cpp @@ -44,6 +44,9 @@ namespace Catch { catch( TestFailureException& ) { std::rethrow_exception(std::current_exception()); } + catch( TestSkipException& ) { + std::rethrow_exception(std::current_exception()); + } catch( std::exception const& ex ) { return ex.what(); } diff --git a/src/catch2/internal/catch_result_type.hpp b/src/catch2/internal/catch_result_type.hpp index f0906fcd30..65d2d4cacd 100644 --- a/src/catch2/internal/catch_result_type.hpp +++ b/src/catch2/internal/catch_result_type.hpp @@ -16,6 +16,8 @@ namespace Catch { Ok = 0, Info = 1, Warning = 2, + // TODO: Should explicit skip be considered "not OK" (cf. isOk)? I.e., should it have the failure bit? + ExplicitSkip = 4, FailureBit = 0x10, diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp index 41d819ba7a..b8227e6791 100644 --- a/src/catch2/internal/catch_run_context.cpp +++ b/src/catch2/internal/catch_run_context.cpp @@ -236,6 +236,9 @@ namespace Catch { if (result.getResultType() == ResultWas::Ok) { m_totals.assertions.passed++; m_lastAssertionPassed = true; + } else if (result.getResultType() == ResultWas::ExplicitSkip) { + m_totals.assertions.skipped++; + m_lastAssertionPassed = true; } else if (!result.succeeded()) { m_lastAssertionPassed = false; if (result.isOk()) { @@ -443,6 +446,8 @@ namespace Catch { duration = timer.getElapsedSeconds(); } CATCH_CATCH_ANON (TestFailureException&) { // This just means the test was aborted due to failure + } CATCH_CATCH_ANON (TestSkipException&) { + // This just means the test was explicitly skipped } CATCH_CATCH_ALL { // Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions // are reported without translation at the point of origin. @@ -534,8 +539,13 @@ namespace Catch { data.message = static_cast(message); AssertionResult assertionResult{ m_lastAssertionInfo, data }; assertionEnded( assertionResult ); - if( !assertionResult.isOk() ) + if ( !assertionResult.isOk() ) { populateReaction( reaction ); + } else if ( resultType == ResultWas::ExplicitSkip ) { + // TODO: Need to handle this explicitly, as ExplicitSkip is + // considered "OK" + reaction.shouldSkip = true; + } } void RunContext::handleUnexpectedExceptionNotThrown( AssertionInfo const& info, diff --git a/src/catch2/internal/catch_test_failure_exception.hpp b/src/catch2/internal/catch_test_failure_exception.hpp index bb6803bae3..31a3a04d7f 100644 --- a/src/catch2/internal/catch_test_failure_exception.hpp +++ b/src/catch2/internal/catch_test_failure_exception.hpp @@ -13,6 +13,9 @@ namespace Catch { //! Used to signal that an assertion macro failed struct TestFailureException{}; + //! Used to signal that the remainder of a test should be skipped + struct TestSkipException{}; + } // namespace Catch #endif // CATCH_TEST_FAILURE_EXCEPTION_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_compact.cpp b/src/catch2/reporters/catch_reporter_compact.cpp index 6c77003f43..5986326f81 100644 --- a/src/catch2/reporters/catch_reporter_compact.cpp +++ b/src/catch2/reporters/catch_reporter_compact.cpp @@ -159,6 +159,7 @@ class AssertionPrinter { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: + case ResultWas::ExplicitSkip: printResultType(Colour::Error, "** internal error **"); break; } diff --git a/src/catch2/reporters/catch_reporter_console.cpp b/src/catch2/reporters/catch_reporter_console.cpp index d5dd89c5a7..8c113a799f 100644 --- a/src/catch2/reporters/catch_reporter_console.cpp +++ b/src/catch2/reporters/catch_reporter_console.cpp @@ -109,6 +109,14 @@ class ConsoleAssertionPrinter { if (_stats.infoMessages.size() > 1) messageLabel = "explicitly with messages"; break; + case ResultWas::ExplicitSkip: + colour = Colour::Skip; + passOrFail = "SKIPPED"_sr; + if (_stats.infoMessages.size() == 1) + messageLabel = "explicitly with message"; + if (_stats.infoMessages.size() > 1) + messageLabel = "explicitly with messages"; + break; // These cases are here to prevent compiler warnings case ResultWas::Unknown: case ResultWas::FailureBit: @@ -183,13 +191,16 @@ std::size_t makeRatio( std::uint64_t number, std::uint64_t total ) { return (ratio == 0 && number > 0) ? 1 : static_cast(ratio); } -std::size_t& findMax( std::size_t& i, std::size_t& j, std::size_t& k ) { - if (i > j && i > k) +std::size_t& +findMax( std::size_t& i, std::size_t& j, std::size_t& k, std::size_t& l ) { + if (i > j && i > k && i > l) return i; - else if (j > k) + else if (j > k && j > l) return j; - else + else if (k > l) return k; + else + return l; } enum class Justification { Left, Right }; @@ -398,7 +409,8 @@ void ConsoleReporter::assertionEnded(AssertionStats const& _assertionStats) { bool includeResults = m_config->includeSuccessfulResults() || !result.isOk(); // Drop out if result was successful but we're not printing them. - if (!includeResults && result.getResultType() != ResultWas::Warning) + // TODO: Make configurable whether skips should be printed + if (!includeResults && result.getResultType() != ResultWas::Warning && result.getResultType() != ResultWas::ExplicitSkip) return; lazyPrint(); @@ -641,6 +653,9 @@ void ConsoleReporter::printTotals( Totals const& totals ) { columns.push_back(SummaryColumn("failed as expected", Colour::ResultExpectedFailure) .addRow(totals.testCases.failedButOk) .addRow(totals.assertions.failedButOk)); + columns.push_back(SummaryColumn("skipped", Colour::Skip) + .addRow(totals.testCases.skipped) + .addRow(totals.assertions.skipped)); printSummaryRow("test cases"_sr, columns, 0); printSummaryRow("assertions"_sr, columns, 1); @@ -669,10 +684,11 @@ void ConsoleReporter::printTotalsDivider(Totals const& totals) { std::size_t failedRatio = makeRatio(totals.testCases.failed, totals.testCases.total()); std::size_t failedButOkRatio = makeRatio(totals.testCases.failedButOk, totals.testCases.total()); std::size_t passedRatio = makeRatio(totals.testCases.passed, totals.testCases.total()); - while (failedRatio + failedButOkRatio + passedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1) - findMax(failedRatio, failedButOkRatio, passedRatio)++; + std::size_t skippedRatio = makeRatio(totals.testCases.skipped, totals.testCases.total()); + while (failedRatio + failedButOkRatio + passedRatio + skippedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1) + findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)++; while (failedRatio + failedButOkRatio + passedRatio > CATCH_CONFIG_CONSOLE_WIDTH - 1) - findMax(failedRatio, failedButOkRatio, passedRatio)--; + findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)--; m_stream << Colour(Colour::Error) << std::string(failedRatio, '='); m_stream << Colour(Colour::ResultExpectedFailure) << std::string(failedButOkRatio, '='); @@ -680,6 +696,7 @@ void ConsoleReporter::printTotalsDivider(Totals const& totals) { m_stream << Colour(Colour::ResultSuccess) << std::string(passedRatio, '='); else m_stream << Colour(Colour::Success) << std::string(passedRatio, '='); + m_stream << Colour(Colour::Skip) << std::string(skippedRatio, '='); } else { m_stream << Colour(Colour::Warning) << std::string(CATCH_CONFIG_CONSOLE_WIDTH - 1, '='); } diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp index e12fc3eaa8..a2db590267 100644 --- a/src/catch2/reporters/catch_reporter_junit.cpp +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -249,6 +249,7 @@ namespace Catch { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: + case ResultWas::ExplicitSkip: elementName = "internalError"; break; } diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp index d45bbfe11c..7420dbe63e 100644 --- a/src/catch2/reporters/catch_reporter_sonarqube.cpp +++ b/src/catch2/reporters/catch_reporter_sonarqube.cpp @@ -105,6 +105,7 @@ namespace Catch { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: + case ResultWas::ExplicitSkip: elementName = "internalError"; break; } diff --git a/src/catch2/reporters/catch_reporter_tap.cpp b/src/catch2/reporters/catch_reporter_tap.cpp index a887ed6c86..d5ae3aae94 100644 --- a/src/catch2/reporters/catch_reporter_tap.cpp +++ b/src/catch2/reporters/catch_reporter_tap.cpp @@ -98,6 +98,7 @@ namespace Catch { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: + case ResultWas::ExplicitSkip: printResultType("** internal error **"_sr); break; } diff --git a/src/catch2/reporters/catch_reporter_teamcity.cpp b/src/catch2/reporters/catch_reporter_teamcity.cpp index 8f16757a1b..0a2713f22f 100644 --- a/src/catch2/reporters/catch_reporter_teamcity.cpp +++ b/src/catch2/reporters/catch_reporter_teamcity.cpp @@ -88,6 +88,7 @@ namespace Catch { case ResultWas::Ok: case ResultWas::Info: case ResultWas::Warning: + case ResultWas::ExplicitSkip: CATCH_ERROR("Internal error in TeamCity reporter"); // These cases are here to prevent compiler warnings case ResultWas::Unknown: diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d78308ed61..d2b2f51cc7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -108,6 +108,7 @@ set(TEST_SOURCES ${SELF_TEST_DIR}/UsageTests/Generators.tests.cpp ${SELF_TEST_DIR}/UsageTests/Message.tests.cpp ${SELF_TEST_DIR}/UsageTests/Misc.tests.cpp + ${SELF_TEST_DIR}/UsageTests/Skip.tests.cpp ${SELF_TEST_DIR}/UsageTests/ToStringByte.tests.cpp ${SELF_TEST_DIR}/UsageTests/ToStringChrono.tests.cpp ${SELF_TEST_DIR}/UsageTests/ToStringGeneral.tests.cpp diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index c575637fae..23b2c50e67 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -298,10 +298,12 @@ Message from section two :test-result: PASS comparisons between const int variables :test-result: PASS comparisons between int variables :test-result: PASS convertToBits +:test-result: XFAIL dynamic skipping works with generators :test-result: PASS empty tags are not allowed :test-result: PASS erfc_inv :test-result: PASS estimate_clock_resolution :test-result: PASS even more nested SECTION tests +:test-result: XFAIL failed assertions before SKIP are still reported :test-result: FAIL first tag loose text artifact :test-result: FAIL has printf @@ -339,9 +341,11 @@ loose text artifact :test-result: PASS run_for_at_least, chronometer :test-result: PASS run_for_at_least, int :test-result: FAIL second tag +:test-result: XFAIL sections can be skipped dynamically at runtime :test-result: FAIL send a single char to INFO :test-result: FAIL sends information to INFO :test-result: PASS shortened hide tags are split apart +:test-result: XFAIL skipped tests can optionally provide a reason :test-result: PASS splitString :test-result: FAIL stacks unscoped info in loops :test-result: PASS startsWith @@ -362,6 +366,7 @@ loose text artifact :test-result: PASS strlen3 :test-result: PASS tables :test-result: PASS tags with dots in later positions are not parsed as hidden +:test-result: XFAIL tests can be skipped dynamically at runtime :test-result: FAIL thrown std::strings are translated :test-result: PASS toString on const wchar_t const pointer returns the string contents :test-result: PASS toString on const wchar_t pointer returns the string contents diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index 31c0b10696..b29bdf2a31 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -291,10 +291,12 @@ :test-result: PASS comparisons between const int variables :test-result: PASS comparisons between int variables :test-result: PASS convertToBits +:test-result: XFAIL dynamic skipping works with generators :test-result: PASS empty tags are not allowed :test-result: PASS erfc_inv :test-result: PASS estimate_clock_resolution :test-result: PASS even more nested SECTION tests +:test-result: XFAIL failed assertions before SKIP are still reported :test-result: FAIL first tag :test-result: FAIL has printf :test-result: PASS is_unary_function @@ -331,9 +333,11 @@ :test-result: PASS run_for_at_least, chronometer :test-result: PASS run_for_at_least, int :test-result: FAIL second tag +:test-result: XFAIL sections can be skipped dynamically at runtime :test-result: FAIL send a single char to INFO :test-result: FAIL sends information to INFO :test-result: PASS shortened hide tags are split apart +:test-result: XFAIL skipped tests can optionally provide a reason :test-result: PASS splitString :test-result: FAIL stacks unscoped info in loops :test-result: PASS startsWith @@ -354,6 +358,7 @@ :test-result: PASS strlen3 :test-result: PASS tables :test-result: PASS tags with dots in later positions are not parsed as hidden +:test-result: XFAIL tests can be skipped dynamically at runtime :test-result: FAIL thrown std::strings are translated :test-result: PASS toString on const wchar_t const pointer returns the string contents :test-result: PASS toString on const wchar_t pointer returns the string contents diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 20098061a9..407345b1ea 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -2189,6 +2189,9 @@ FloatingPoint.tests.cpp:: passed: convertToBits( -0. ) == ( 1ULL << 9223372036854775808 (0x) FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: Tag.tests.cpp:: passed: Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.103560) == Approx(-0.09203687623843015) for: -0.0920368762 == Approx( -0.0920368762 ) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.067400) == Approx(-0.05980291115763361) for: -0.0598029112 == Approx( -0.0598029112 ) @@ -2198,6 +2201,8 @@ InternalBenchmark.tests.cpp:: passed: res.outliers.total() == 0 for Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: +Skip.tests.cpp:: failed: 3 == 4 +Skip.tests.cpp:: ** internal error **: loose text artifact Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -2337,9 +2342,12 @@ InternalBenchmark.tests.cpp:: passed: x >= old_x for: 128 >= 64 InternalBenchmark.tests.cpp:: passed: Timing.elapsed >= time for: 128 ns >= 100 ns InternalBenchmark.tests.cpp:: passed: Timing.result == Timing.iterations + 17 for: 145 == 145 InternalBenchmark.tests.cpp:: passed: Timing.iterations >= time.count() for: 128 >= 100 +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: Misc.tests.cpp:: failed: false with 1 message: '3' Message.tests.cpp:: failed: false with 2 messages: 'hi' and 'i := 7' Tag.tests.cpp:: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +Skip.tests.cpp:: ** internal error **: StringManip.tests.cpp:: passed: splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } StringManip.tests.cpp:: passed: splitStringRef("abc", ','), Equals(std::vector{"abc"}) for: { abc } Equals: { abc } StringManip.tests.cpp:: passed: splitStringRef("abc,def", ','), Equals(std::vector{"abc", "def"}) for: { abc, def } Equals: { abc, def } @@ -2403,6 +2411,7 @@ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_cast(std::get<1>(data)) for: 6 == 6 Tag.tests.cpp:: passed: testcase.tags.size() == 1 for: 1 == 1 Tag.tests.cpp:: passed: testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag +Skip.tests.cpp:: ** internal error **: Exception.tests.cpp:: failed: unexpected exception with message: 'Why would you throw a std::string?' Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 993a7b7cd6..d293611454 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -2182,6 +2182,9 @@ FloatingPoint.tests.cpp:: passed: convertToBits( -0. ) == ( 1ULL << 9223372036854775808 (0x) FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: Tag.tests.cpp:: passed: Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.103560) == Approx(-0.09203687623843015) for: -0.0920368762 == Approx( -0.0920368762 ) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.067400) == Approx(-0.05980291115763361) for: -0.0598029112 == Approx( -0.0598029112 ) @@ -2191,6 +2194,8 @@ InternalBenchmark.tests.cpp:: passed: res.outliers.total() == 0 for Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: +Skip.tests.cpp:: failed: 3 == 4 +Skip.tests.cpp:: ** internal error **: Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -2329,9 +2334,12 @@ InternalBenchmark.tests.cpp:: passed: x >= old_x for: 128 >= 64 InternalBenchmark.tests.cpp:: passed: Timing.elapsed >= time for: 128 ns >= 100 ns InternalBenchmark.tests.cpp:: passed: Timing.result == Timing.iterations + 17 for: 145 == 145 InternalBenchmark.tests.cpp:: passed: Timing.iterations >= time.count() for: 128 >= 100 +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: Misc.tests.cpp:: failed: false with 1 message: '3' Message.tests.cpp:: failed: false with 2 messages: 'hi' and 'i := 7' Tag.tests.cpp:: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +Skip.tests.cpp:: ** internal error **: StringManip.tests.cpp:: passed: splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } StringManip.tests.cpp:: passed: splitStringRef("abc", ','), Equals(std::vector{"abc"}) for: { abc } Equals: { abc } StringManip.tests.cpp:: passed: splitStringRef("abc,def", ','), Equals(std::vector{"abc", "def"}) for: { abc, def } Equals: { abc, def } @@ -2395,6 +2403,7 @@ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_cast(std::get<1>(data)) for: 6 == 6 Tag.tests.cpp:: passed: testcase.tags.size() == 1 for: 1 == 1 Tag.tests.cpp:: passed: testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag +Skip.tests.cpp:: ** internal error **: Exception.tests.cpp:: failed: unexpected exception with message: 'Why would you throw a std::string?' Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index f6f6b2922c..3ac939cd0c 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1187,6 +1187,37 @@ Misc.tests.cpp:: FAILED: with expansion: false +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 41 + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + +------------------------------------------------------------------------------- +failed assertions before SKIP are still reported +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + CHECK( 3 == 4 ) + +Skip.tests.cpp:: SKIPPED: + loose text artifact ------------------------------------------------------------------------------- just failure @@ -1339,6 +1370,15 @@ Message.tests.cpp:: FAILED: with message: this SHOULD be seen only ONCE +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- send a single char to INFO ------------------------------------------------------------------------------- @@ -1362,6 +1402,16 @@ with messages: hi i := 7 +------------------------------------------------------------------------------- +skipped tests can optionally provide a reason +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- stacks unscoped info in loops ------------------------------------------------------------------------------- @@ -1384,6 +1434,14 @@ with messages: 5 6 +------------------------------------------------------------------------------- +tests can be skipped dynamically at runtime +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- thrown std::strings are translated ------------------------------------------------------------------------------- @@ -1395,6 +1453,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 381 | 305 passed | 69 failed | 7 failed as expected -assertions: 2209 | 2054 passed | 128 failed | 27 failed as expected +test cases: 386 | 305 passed | 69 failed | 8 failed as expected | 4 skipped +assertions: 2218 | 2056 passed | 128 failed | 28 failed as expected | 6 skipped diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 8d1a25b437..54e5a341d3 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -15448,6 +15448,34 @@ FloatingPoint.tests.cpp:: PASSED: with expansion: 1 == 1 +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 41 + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- empty tags are not allowed ------------------------------------------------------------------------------- @@ -15523,6 +15551,17 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: +------------------------------------------------------------------------------- +failed assertions before SKIP are still reported +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + CHECK( 3 == 4 ) + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- first tag ------------------------------------------------------------------------------- @@ -16602,6 +16641,24 @@ Misc.tests.cpp: No assertions in test case 'second tag' +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + not skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- send a single char to INFO ------------------------------------------------------------------------------- @@ -16636,6 +16693,16 @@ Tag.tests.cpp:: PASSED: with expansion: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +------------------------------------------------------------------------------- +skipped tests can optionally provide a reason +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- splitString ------------------------------------------------------------------------------- @@ -17047,6 +17114,14 @@ Tag.tests.cpp:: PASSED: with expansion: magic.tag == magic.tag +------------------------------------------------------------------------------- +tests can be skipped dynamically at runtime +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- thrown std::strings are translated ------------------------------------------------------------------------------- @@ -17754,6 +17829,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 381 | 291 passed | 83 failed | 7 failed as expected -assertions: 2224 | 2054 passed | 143 failed | 27 failed as expected +test cases: 386 | 291 passed | 83 failed | 8 failed as expected | 4 skipped +assertions: 2233 | 2056 passed | 143 failed | 28 failed as expected | 6 skipped diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 920a1c7ad6..ce79a27ac8 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -15441,6 +15441,34 @@ FloatingPoint.tests.cpp:: PASSED: with expansion: 1 == 1 +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 41 + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- empty tags are not allowed ------------------------------------------------------------------------------- @@ -15516,6 +15544,17 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: +------------------------------------------------------------------------------- +failed assertions before SKIP are still reported +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + CHECK( 3 == 4 ) + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- first tag ------------------------------------------------------------------------------- @@ -16594,6 +16633,24 @@ Misc.tests.cpp: No assertions in test case 'second tag' +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + not skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- send a single char to INFO ------------------------------------------------------------------------------- @@ -16628,6 +16685,16 @@ Tag.tests.cpp:: PASSED: with expansion: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +------------------------------------------------------------------------------- +skipped tests can optionally provide a reason +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- splitString ------------------------------------------------------------------------------- @@ -17039,6 +17106,14 @@ Tag.tests.cpp:: PASSED: with expansion: magic.tag == magic.tag +------------------------------------------------------------------------------- +tests can be skipped dynamically at runtime +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- thrown std::strings are translated ------------------------------------------------------------------------------- @@ -17746,6 +17821,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 381 | 291 passed | 83 failed | 7 failed as expected -assertions: 2224 | 2054 passed | 143 failed | 27 failed as expected +test cases: 386 | 291 passed | 83 failed | 8 failed as expected | 4 skipped +assertions: 2233 | 2056 passed | 143 failed | 28 failed as expected | 6 skipped diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index c020fa4bd3..b1c9f3d1ea 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1619,12 +1619,21 @@ Misc.tests.cpp: + + + + +FAILED: + CHECK( 3 == 4 ) +Skip.tests.cpp: + + @@ -1786,6 +1795,8 @@ Message.tests.cpp: + + FAILED: @@ -1804,6 +1815,7 @@ Message.tests.cpp: + @@ -1847,6 +1859,7 @@ Message.tests.cpp: + FAILED: diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index 913983395f..b365ea8c44 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -1618,12 +1618,21 @@ Misc.tests.cpp: + + + + +FAILED: + CHECK( 3 == 4 ) +Skip.tests.cpp: + + @@ -1785,6 +1794,8 @@ Message.tests.cpp: + + FAILED: @@ -1803,6 +1814,7 @@ Message.tests.cpp: + @@ -1846,6 +1858,7 @@ Message.tests.cpp: + FAILED: diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index cc901e2999..baabfedca6 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -1812,6 +1812,20 @@ Misc.tests.cpp: + + + + +FAILED: + CHECK( 3 == 4 ) +Skip.tests.cpp: + + + + + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 0359fa40ab..e13490547d 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -1811,6 +1811,20 @@ Misc.tests.cpp: + + + + +FAILED: + CHECK( 3 == 4 ) +Skip.tests.cpp: + + + + + + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index bd0e988100..5ad03cbd52 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -3907,6 +3907,12 @@ ok {test-number} - convertToBits( -0. ) == ( 1ULL << 63 ) for: 92233720368547758 ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 # convertToBits ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 +# dynamic skipping works with generators +** internal error ** 1951 - +# dynamic skipping works with generators +ok {test-number} - +# dynamic skipping works with generators +** internal error ** 1953 - # empty tags are not allowed ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) # erfc_inv @@ -3925,6 +3931,10 @@ ok {test-number} - ok {test-number} - # even more nested SECTION tests ok {test-number} - +# failed assertions before SKIP are still reported +not ok {test-number} - 3 == 4 +# failed assertions before SKIP are still reported +** internal error ** 1964 - loose text artifact # is_unary_function ok {test-number} - with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -4194,12 +4204,18 @@ ok {test-number} - Timing.elapsed >= time for: 128 ns >= 100 ns ok {test-number} - Timing.result == Timing.iterations + 17 for: 145 == 145 # run_for_at_least, int ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100 +# sections can be skipped dynamically at runtime +ok {test-number} - +# sections can be skipped dynamically at runtime +** internal error ** 2100 - # send a single char to INFO not ok {test-number} - false with 1 message: '3' # sends information to INFO not ok {test-number} - false with 2 messages: 'hi' and 'i := 7' # shortened hide tags are split apart ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +# skipped tests can optionally provide a reason +** internal error ** 2104 - # splitString ok {test-number} - splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } # splitString @@ -4278,6 +4294,8 @@ ok {test-number} - strlen(std::get<0>(data)) == static_cast(std::get<1>( ok {test-number} - testcase.tags.size() == 1 for: 1 == 1 # tags with dots in later positions are not parsed as hidden ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag +# tests can be skipped dynamically at runtime +** internal error ** 2144 - # thrown std::strings are translated not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?' # toString on const wchar_t const pointer returns the string contents @@ -4450,5 +4468,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2224 +1..2233 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 59e8d1a869..85de7416b6 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -3900,6 +3900,12 @@ ok {test-number} - convertToBits( -0. ) == ( 1ULL << 63 ) for: 92233720368547758 ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 # convertToBits ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 +# dynamic skipping works with generators +** internal error ** 1951 - +# dynamic skipping works with generators +ok {test-number} - +# dynamic skipping works with generators +** internal error ** 1953 - # empty tags are not allowed ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) # erfc_inv @@ -3918,6 +3924,10 @@ ok {test-number} - ok {test-number} - # even more nested SECTION tests ok {test-number} - +# failed assertions before SKIP are still reported +not ok {test-number} - 3 == 4 +# failed assertions before SKIP are still reported +** internal error ** 1964 - # is_unary_function ok {test-number} - with 1 message: 'Catch::Clara::Detail::is_unary_function::value' # is_unary_function @@ -4186,12 +4196,18 @@ ok {test-number} - Timing.elapsed >= time for: 128 ns >= 100 ns ok {test-number} - Timing.result == Timing.iterations + 17 for: 145 == 145 # run_for_at_least, int ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100 +# sections can be skipped dynamically at runtime +ok {test-number} - +# sections can be skipped dynamically at runtime +** internal error ** 2100 - # send a single char to INFO not ok {test-number} - false with 1 message: '3' # sends information to INFO not ok {test-number} - false with 2 messages: 'hi' and 'i := 7' # shortened hide tags are split apart ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +# skipped tests can optionally provide a reason +** internal error ** 2104 - # splitString ok {test-number} - splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } # splitString @@ -4270,6 +4286,8 @@ ok {test-number} - strlen(std::get<0>(data)) == static_cast(std::get<1>( ok {test-number} - testcase.tags.size() == 1 for: 1 == 1 # tags with dots in later positions are not parsed as hidden ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag +# tests can be skipped dynamically at runtime +** internal error ** 2144 - # thrown std::strings are translated not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?' # toString on const wchar_t const pointer returns the string contents @@ -4442,5 +4460,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2224 +1..2233 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index e7279bc4f5..9e6b926c53 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -726,6 +726,8 @@ Misc.tests.cpp:|nexpression failed|n REQUIRE( testCheckedIf( false ##teamcity[testFinished name='comparisons between int variables' duration="{duration}"] ##teamcity[testStarted name='convertToBits'] ##teamcity[testFinished name='convertToBits' duration="{duration}"] +##teamcity[testStarted name='dynamic skipping works with generators'] +##teamcity[testFinished name='dynamic skipping works with generators' duration="{duration}"] ##teamcity[testStarted name='empty tags are not allowed'] ##teamcity[testFinished name='empty tags are not allowed' duration="{duration}"] ##teamcity[testStarted name='erfc_inv'] @@ -734,6 +736,9 @@ Misc.tests.cpp:|nexpression failed|n REQUIRE( testCheckedIf( false ##teamcity[testFinished name='estimate_clock_resolution' duration="{duration}"] ##teamcity[testStarted name='even more nested SECTION tests'] ##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"] +##teamcity[testStarted name='failed assertions before SKIP are still reported'] +Skip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failed assertions before SKIP are still reported' duration="{duration}"] ##teamcity[testStarted name='first tag'] ##teamcity[testFinished name='first tag' duration="{duration}"] ##teamcity[testStarted name='has printf'] @@ -821,6 +826,8 @@ Message.tests.cpp:|nexpression failed with message:|n "this SHOULD ##teamcity[testFinished name='run_for_at_least, int' duration="{duration}"] ##teamcity[testStarted name='second tag'] ##teamcity[testFinished name='second tag' duration="{duration}"] +##teamcity[testStarted name='sections can be skipped dynamically at runtime'] +##teamcity[testFinished name='sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='send a single char to INFO'] Misc.tests.cpp:|nexpression failed with message:|n "3"|n REQUIRE( false )|nwith expansion:|n false|n'] ##teamcity[testFinished name='send a single char to INFO' duration="{duration}"] @@ -829,6 +836,8 @@ Message.tests.cpp:|nexpression failed with messages:|n "hi"|n "i ##teamcity[testFinished name='sends information to INFO' duration="{duration}"] ##teamcity[testStarted name='shortened hide tags are split apart'] ##teamcity[testFinished name='shortened hide tags are split apart' duration="{duration}"] +##teamcity[testStarted name='skipped tests can optionally provide a reason'] +##teamcity[testFinished name='skipped tests can optionally provide a reason' duration="{duration}"] ##teamcity[testStarted name='splitString'] ##teamcity[testFinished name='splitString' duration="{duration}"] ##teamcity[testStarted name='stacks unscoped info in loops'] @@ -871,6 +880,8 @@ Message.tests.cpp:|nexpression failed with messages:|n "Count 4 to ##teamcity[testFinished name='tables' duration="{duration}"] ##teamcity[testStarted name='tags with dots in later positions are not parsed as hidden'] ##teamcity[testFinished name='tags with dots in later positions are not parsed as hidden' duration="{duration}"] +##teamcity[testStarted name='tests can be skipped dynamically at runtime'] +##teamcity[testFinished name='tests can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='thrown std::strings are translated'] Exception.tests.cpp:|nunexpected exception with message:|n "Why would you throw a std::string?"'] ##teamcity[testFinished name='thrown std::strings are translated' duration="{duration}"] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index 28912d91db..caa1a1def2 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -726,6 +726,8 @@ Misc.tests.cpp:|nexpression failed|n REQUIRE( testCheckedIf( false ##teamcity[testFinished name='comparisons between int variables' duration="{duration}"] ##teamcity[testStarted name='convertToBits'] ##teamcity[testFinished name='convertToBits' duration="{duration}"] +##teamcity[testStarted name='dynamic skipping works with generators'] +##teamcity[testFinished name='dynamic skipping works with generators' duration="{duration}"] ##teamcity[testStarted name='empty tags are not allowed'] ##teamcity[testFinished name='empty tags are not allowed' duration="{duration}"] ##teamcity[testStarted name='erfc_inv'] @@ -734,6 +736,9 @@ Misc.tests.cpp:|nexpression failed|n REQUIRE( testCheckedIf( false ##teamcity[testFinished name='estimate_clock_resolution' duration="{duration}"] ##teamcity[testStarted name='even more nested SECTION tests'] ##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"] +##teamcity[testStarted name='failed assertions before SKIP are still reported'] +Skip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failed assertions before SKIP are still reported' duration="{duration}"] ##teamcity[testStarted name='first tag'] ##teamcity[testFinished name='first tag' duration="{duration}"] ##teamcity[testStarted name='has printf'] @@ -820,6 +825,8 @@ Message.tests.cpp:|nexpression failed with message:|n "this SHOULD ##teamcity[testFinished name='run_for_at_least, int' duration="{duration}"] ##teamcity[testStarted name='second tag'] ##teamcity[testFinished name='second tag' duration="{duration}"] +##teamcity[testStarted name='sections can be skipped dynamically at runtime'] +##teamcity[testFinished name='sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='send a single char to INFO'] Misc.tests.cpp:|nexpression failed with message:|n "3"|n REQUIRE( false )|nwith expansion:|n false|n'] ##teamcity[testFinished name='send a single char to INFO' duration="{duration}"] @@ -828,6 +835,8 @@ Message.tests.cpp:|nexpression failed with messages:|n "hi"|n "i ##teamcity[testFinished name='sends information to INFO' duration="{duration}"] ##teamcity[testStarted name='shortened hide tags are split apart'] ##teamcity[testFinished name='shortened hide tags are split apart' duration="{duration}"] +##teamcity[testStarted name='skipped tests can optionally provide a reason'] +##teamcity[testFinished name='skipped tests can optionally provide a reason' duration="{duration}"] ##teamcity[testStarted name='splitString'] ##teamcity[testFinished name='splitString' duration="{duration}"] ##teamcity[testStarted name='stacks unscoped info in loops'] @@ -870,6 +879,8 @@ Message.tests.cpp:|nexpression failed with messages:|n "Count 4 to ##teamcity[testFinished name='tables' duration="{duration}"] ##teamcity[testStarted name='tags with dots in later positions are not parsed as hidden'] ##teamcity[testFinished name='tags with dots in later positions are not parsed as hidden' duration="{duration}"] +##teamcity[testStarted name='tests can be skipped dynamically at runtime'] +##teamcity[testFinished name='tests can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='thrown std::strings are translated'] Exception.tests.cpp:|nunexpected exception with message:|n "Why would you throw a std::string?"'] ##teamcity[testFinished name='thrown std::strings are translated' duration="{duration}"] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 49b1148450..141ddfa02e 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -18288,6 +18288,9 @@ There is no extra whitespace here + + + @@ -18363,6 +18366,17 @@ There is no extra whitespace here + + + + 3 == 4 + + + 3 == 4 + + + + @@ -19583,6 +19597,15 @@ loose text artifact + +
+ +
+
+ +
+ + 3 @@ -19625,6 +19648,9 @@ loose text artifact + + + @@ -20063,6 +20089,9 @@ loose text artifact + + + Why would you throw a std::string? @@ -20863,6 +20892,6 @@ loose text artifact - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index f73fe121e4..f3c8376a58 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -18288,6 +18288,9 @@ There is no extra whitespace here + + + @@ -18363,6 +18366,17 @@ There is no extra whitespace here + + + + 3 == 4 + + + 3 == 4 + + + + @@ -19582,6 +19596,15 @@ There is no extra whitespace here + +
+ +
+
+ +
+ + 3 @@ -19624,6 +19647,9 @@ There is no extra whitespace here + + + @@ -20062,6 +20088,9 @@ There is no extra whitespace here + + + Why would you throw a std::string? @@ -20862,6 +20891,6 @@ There is no extra whitespace here - - + + diff --git a/tests/SelfTest/UsageTests/Skip.tests.cpp b/tests/SelfTest/UsageTests/Skip.tests.cpp new file mode 100644 index 0000000000..dee884bc42 --- /dev/null +++ b/tests/SelfTest/UsageTests/Skip.tests.cpp @@ -0,0 +1,39 @@ +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include +#include + +TEST_CASE( "tests can be skipped dynamically at runtime", "[skipping]" ) { + SKIP(); + FAIL( "this is not reached" ); +} + +TEST_CASE( "skipped tests can optionally provide a reason", "[skipping]" ) { + const int answer = 43; + SKIP( "skipping because answer = " << answer ); + FAIL( "this is not reached" ); +} + +TEST_CASE( "sections can be skipped dynamically at runtime", "[skipping]" ) { + SECTION( "not skipped" ) { SUCCEED(); } + SECTION( "skipped" ) { SKIP(); } +} + +TEST_CASE( "dynamic skipping works with generators", "[skipping]" ) { + const int answer = GENERATE( 41, 42, 43 ); + if ( answer != 42 ) { + SKIP( "skipping because answer = " << answer ); + } + SUCCEED(); +} + +TEST_CASE( "failed assertions before SKIP are still reported", + "[skipping][!shouldfail]" ) { + CHECK( 3 == 4 ); + SKIP(); +}