From add702c752333e18364ffa77ffcb52e23c1e2696 Mon Sep 17 00:00:00 2001 From: Philip Salzmann Date: Mon, 31 Jan 2022 16:34:27 +0100 Subject: [PATCH 1/9] Add new SKIP macro for skipping tests at runtime This adds a new `SKIP` macro for dynamically skipping tests at runtime. The "skipped" status of a test case is treated as a first-class citizen, like "succeeded" or "failed", and is reported with a new color on the console. --- src/catch2/catch_test_macros.hpp | 4 + src/catch2/catch_totals.cpp | 8 +- src/catch2/catch_totals.hpp | 1 + .../internal/catch_assertion_handler.cpp | 7 ++ .../internal/catch_assertion_handler.hpp | 1 + src/catch2/internal/catch_console_colour.hpp | 1 + .../catch_exception_translator_registry.cpp | 3 + src/catch2/internal/catch_result_type.hpp | 2 + src/catch2/internal/catch_run_context.cpp | 12 ++- .../internal/catch_test_failure_exception.hpp | 3 + .../reporters/catch_reporter_compact.cpp | 1 + .../reporters/catch_reporter_console.cpp | 31 ++++++-- .../reporters/catch_reporter_helpers.cpp | 3 + src/catch2/reporters/catch_reporter_junit.cpp | 1 + .../reporters/catch_reporter_sonarqube.cpp | 1 + src/catch2/reporters/catch_reporter_tap.cpp | 1 + .../reporters/catch_reporter_teamcity.cpp | 1 + tests/CMakeLists.txt | 1 + .../Baselines/automake.sw.approved.txt | 5 ++ .../Baselines/automake.sw.multi.approved.txt | 5 ++ .../Baselines/compact.sw.approved.txt | 13 ++- .../Baselines/compact.sw.multi.approved.txt | 13 ++- .../Baselines/console.std.approved.txt | 62 ++++++++++++++- .../Baselines/console.sw.approved.txt | 79 ++++++++++++++++++- .../Baselines/console.sw.multi.approved.txt | 79 ++++++++++++++++++- .../SelfTest/Baselines/junit.sw.approved.txt | 15 +++- .../Baselines/junit.sw.multi.approved.txt | 15 +++- .../Baselines/sonarqube.sw.approved.txt | 14 ++++ .../Baselines/sonarqube.sw.multi.approved.txt | 14 ++++ tests/SelfTest/Baselines/tap.sw.approved.txt | 20 ++++- .../Baselines/tap.sw.multi.approved.txt | 20 ++++- .../Baselines/teamcity.sw.approved.txt | 11 +++ .../Baselines/teamcity.sw.multi.approved.txt | 11 +++ tests/SelfTest/Baselines/xml.sw.approved.txt | 33 +++++++- .../Baselines/xml.sw.multi.approved.txt | 33 +++++++- tests/SelfTest/UsageTests/Skip.tests.cpp | 38 +++++++++ 36 files changed, 533 insertions(+), 29 deletions(-) create mode 100644 tests/SelfTest/UsageTests/Skip.tests.cpp diff --git a/src/catch2/catch_test_macros.hpp b/src/catch2/catch_test_macros.hpp index cce2852f83..1088afbedd 100644 --- a/src/catch2/catch_test_macros.hpp +++ b/src/catch2/catch_test_macros.hpp @@ -49,6 +49,7 @@ #define CATCH_FAIL( ... ) INTERNAL_CATCH_MSG( "CATCH_FAIL", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, __VA_ARGS__ ) #define CATCH_FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( "CATCH_FAIL_CHECK", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) #define CATCH_SUCCEED( ... ) INTERNAL_CATCH_MSG( "CATCH_SUCCEED", Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) + #define CATCH_SKIP( ... ) INTERNAL_CATCH_MSG( "SKIP", Catch::ResultWas::ExplicitSkip, Catch::ResultDisposition::Normal, __VA_ARGS__ ) #if !defined(CATCH_CONFIG_RUNTIME_STATIC_REQUIRE) @@ -102,6 +103,7 @@ #define CATCH_FAIL( ... ) (void)(0) #define CATCH_FAIL_CHECK( ... ) (void)(0) #define CATCH_SUCCEED( ... ) (void)(0) + #define CATCH_SKIP( ... ) (void)(0) #define CATCH_STATIC_REQUIRE( ... ) (void)(0) #define CATCH_STATIC_REQUIRE_FALSE( ... ) (void)(0) @@ -146,6 +148,7 @@ #define FAIL( ... ) INTERNAL_CATCH_MSG( "FAIL", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, __VA_ARGS__ ) #define FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( "FAIL_CHECK", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) #define SUCCEED( ... ) INTERNAL_CATCH_MSG( "SUCCEED", Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) + #define SKIP( ... ) INTERNAL_CATCH_MSG( "SKIP", Catch::ResultWas::ExplicitSkip, Catch::ResultDisposition::Normal, __VA_ARGS__ ) #if !defined(CATCH_CONFIG_RUNTIME_STATIC_REQUIRE) @@ -198,6 +201,7 @@ #define FAIL( ... ) (void)(0) #define FAIL_CHECK( ... ) (void)(0) #define SUCCEED( ... ) (void)(0) + #define SKIP( ... ) (void)(0) #define STATIC_REQUIRE( ... ) (void)(0) #define STATIC_REQUIRE_FALSE( ... ) (void)(0) diff --git a/src/catch2/catch_totals.cpp b/src/catch2/catch_totals.cpp index a3e2b384f1..bd1954fb9e 100644 --- a/src/catch2/catch_totals.cpp +++ b/src/catch2/catch_totals.cpp @@ -14,6 +14,7 @@ namespace Catch { diff.passed = passed - other.passed; diff.failed = failed - other.failed; diff.failedButOk = failedButOk - other.failedButOk; + diff.skipped = skipped - other.skipped; return diff; } @@ -21,14 +22,15 @@ namespace Catch { passed += other.passed; failed += other.failed; failedButOk += other.failedButOk; + skipped += other.skipped; return *this; } std::uint64_t Counts::total() const { - return passed + failed + failedButOk; + return passed + failed + failedButOk + skipped; } bool Counts::allPassed() const { - return failed == 0 && failedButOk == 0; + return failed == 0 && failedButOk == 0 && skipped == 0; } bool Counts::allOk() const { return failed == 0; @@ -53,6 +55,8 @@ namespace Catch { ++diff.testCases.failed; else if( diff.assertions.failedButOk > 0 ) ++diff.testCases.failedButOk; + else if ( diff.assertions.skipped > 0 ) + ++ diff.testCases.skipped; else ++diff.testCases.passed; return diff; diff --git a/src/catch2/catch_totals.hpp b/src/catch2/catch_totals.hpp index 8dd360c6d1..386392c91f 100644 --- a/src/catch2/catch_totals.hpp +++ b/src/catch2/catch_totals.hpp @@ -23,6 +23,7 @@ namespace Catch { std::uint64_t passed = 0; std::uint64_t failed = 0; std::uint64_t failedButOk = 0; + std::uint64_t skipped = 0; }; struct Totals { diff --git a/src/catch2/internal/catch_assertion_handler.cpp b/src/catch2/internal/catch_assertion_handler.cpp index f051314cce..0b14e0bba0 100644 --- a/src/catch2/internal/catch_assertion_handler.cpp +++ b/src/catch2/internal/catch_assertion_handler.cpp @@ -50,6 +50,13 @@ namespace Catch { if (m_reaction.shouldThrow) { throw_test_failure_exception(); } + if ( m_reaction.shouldSkip ) { +#if !defined( CATCH_CONFIG_DISABLE_EXCEPTIONS ) + throw Catch::TestSkipException(); +#else + CATCH_ERROR( "Explicitly skipping tests during runtime requires exceptions" ); +#endif + } } void AssertionHandler::setCompleted() { m_completed = true; diff --git a/src/catch2/internal/catch_assertion_handler.hpp b/src/catch2/internal/catch_assertion_handler.hpp index 36b55243bf..ae7776d8b6 100644 --- a/src/catch2/internal/catch_assertion_handler.hpp +++ b/src/catch2/internal/catch_assertion_handler.hpp @@ -22,6 +22,7 @@ namespace Catch { struct AssertionReaction { bool shouldDebugBreak = false; bool shouldThrow = false; + bool shouldSkip = false; }; class AssertionHandler { diff --git a/src/catch2/internal/catch_console_colour.hpp b/src/catch2/internal/catch_console_colour.hpp index 9aa6a163b3..8ee808cc2c 100644 --- a/src/catch2/internal/catch_console_colour.hpp +++ b/src/catch2/internal/catch_console_colour.hpp @@ -47,6 +47,7 @@ namespace Catch { Error = BrightRed, Success = Green, + Skip = Blue, OriginalExpression = Cyan, ReconstructedExpression = BrightYellow, diff --git a/src/catch2/internal/catch_exception_translator_registry.cpp b/src/catch2/internal/catch_exception_translator_registry.cpp index 2a240a9b6e..0645c6ce0c 100644 --- a/src/catch2/internal/catch_exception_translator_registry.cpp +++ b/src/catch2/internal/catch_exception_translator_registry.cpp @@ -44,6 +44,9 @@ namespace Catch { catch( TestFailureException& ) { std::rethrow_exception(std::current_exception()); } + catch( TestSkipException& ) { + std::rethrow_exception(std::current_exception()); + } catch( std::exception const& ex ) { return ex.what(); } diff --git a/src/catch2/internal/catch_result_type.hpp b/src/catch2/internal/catch_result_type.hpp index faf0683db0..e66afaff00 100644 --- a/src/catch2/internal/catch_result_type.hpp +++ b/src/catch2/internal/catch_result_type.hpp @@ -16,6 +16,8 @@ namespace Catch { Ok = 0, Info = 1, Warning = 2, + // TODO: Should explicit skip be considered "not OK" (cf. isOk)? I.e., should it have the failure bit? + ExplicitSkip = 4, FailureBit = 0x10, diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp index d2e8fb8c5b..e1b81d0bfe 100644 --- a/src/catch2/internal/catch_run_context.cpp +++ b/src/catch2/internal/catch_run_context.cpp @@ -270,6 +270,9 @@ namespace Catch { if (result.getResultType() == ResultWas::Ok) { m_totals.assertions.passed++; m_lastAssertionPassed = true; + } else if (result.getResultType() == ResultWas::ExplicitSkip) { + m_totals.assertions.skipped++; + m_lastAssertionPassed = true; } else if (!result.succeeded()) { m_lastAssertionPassed = false; if (result.isOk()) { @@ -475,6 +478,8 @@ namespace Catch { duration = timer.getElapsedSeconds(); } CATCH_CATCH_ANON (TestFailureException&) { // This just means the test was aborted due to failure + } CATCH_CATCH_ANON (TestSkipException&) { + // This just means the test was explicitly skipped } CATCH_CATCH_ALL { // Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions // are reported without translation at the point of origin. @@ -571,8 +576,13 @@ namespace Catch { data.message = static_cast(message); AssertionResult assertionResult{ m_lastAssertionInfo, data }; assertionEnded( assertionResult ); - if( !assertionResult.isOk() ) + if ( !assertionResult.isOk() ) { populateReaction( reaction ); + } else if ( resultType == ResultWas::ExplicitSkip ) { + // TODO: Need to handle this explicitly, as ExplicitSkip is + // considered "OK" + reaction.shouldSkip = true; + } } void RunContext::handleUnexpectedExceptionNotThrown( AssertionInfo const& info, diff --git a/src/catch2/internal/catch_test_failure_exception.hpp b/src/catch2/internal/catch_test_failure_exception.hpp index 810a81c9a3..13c5fc082b 100644 --- a/src/catch2/internal/catch_test_failure_exception.hpp +++ b/src/catch2/internal/catch_test_failure_exception.hpp @@ -20,6 +20,9 @@ namespace Catch { */ [[noreturn]] void throw_test_failure_exception(); + //! Used to signal that the remainder of a test should be skipped + struct TestSkipException{}; + } // namespace Catch #endif // CATCH_TEST_FAILURE_EXCEPTION_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_compact.cpp b/src/catch2/reporters/catch_reporter_compact.cpp index d8088457b1..53c3ea53b2 100644 --- a/src/catch2/reporters/catch_reporter_compact.cpp +++ b/src/catch2/reporters/catch_reporter_compact.cpp @@ -109,6 +109,7 @@ class AssertionPrinter { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: + case ResultWas::ExplicitSkip: printResultType(Colour::Error, "** internal error **"); break; } diff --git a/src/catch2/reporters/catch_reporter_console.cpp b/src/catch2/reporters/catch_reporter_console.cpp index 0edb121e61..b394d2bebd 100644 --- a/src/catch2/reporters/catch_reporter_console.cpp +++ b/src/catch2/reporters/catch_reporter_console.cpp @@ -111,6 +111,14 @@ class ConsoleAssertionPrinter { if (_stats.infoMessages.size() > 1) messageLabel = "explicitly with messages"; break; + case ResultWas::ExplicitSkip: + colour = Colour::Skip; + passOrFail = "SKIPPED"_sr; + if (_stats.infoMessages.size() == 1) + messageLabel = "explicitly with message"; + if (_stats.infoMessages.size() > 1) + messageLabel = "explicitly with messages"; + break; // These cases are here to prevent compiler warnings case ResultWas::Unknown: case ResultWas::FailureBit: @@ -185,13 +193,16 @@ std::size_t makeRatio( std::uint64_t number, std::uint64_t total ) { return (ratio == 0 && number > 0) ? 1 : static_cast(ratio); } -std::size_t& findMax( std::size_t& i, std::size_t& j, std::size_t& k ) { - if (i > j && i > k) +std::size_t& +findMax( std::size_t& i, std::size_t& j, std::size_t& k, std::size_t& l ) { + if (i > j && i > k && i > l) return i; - else if (j > k) + else if (j > k && j > l) return j; - else + else if (k > l) return k; + else + return l; } enum class Justification { Left, Right }; @@ -400,7 +411,8 @@ void ConsoleReporter::assertionEnded(AssertionStats const& _assertionStats) { bool includeResults = m_config->includeSuccessfulResults() || !result.isOk(); // Drop out if result was successful but we're not printing them. - if (!includeResults && result.getResultType() != ResultWas::Warning) + // TODO: Make configurable whether skips should be printed + if (!includeResults && result.getResultType() != ResultWas::Warning && result.getResultType() != ResultWas::ExplicitSkip) return; lazyPrint(); @@ -603,10 +615,11 @@ void ConsoleReporter::printTotalsDivider(Totals const& totals) { std::size_t failedRatio = makeRatio(totals.testCases.failed, totals.testCases.total()); std::size_t failedButOkRatio = makeRatio(totals.testCases.failedButOk, totals.testCases.total()); std::size_t passedRatio = makeRatio(totals.testCases.passed, totals.testCases.total()); - while (failedRatio + failedButOkRatio + passedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1) - findMax(failedRatio, failedButOkRatio, passedRatio)++; + std::size_t skippedRatio = makeRatio(totals.testCases.skipped, totals.testCases.total()); + while (failedRatio + failedButOkRatio + passedRatio + skippedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1) + findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)++; while (failedRatio + failedButOkRatio + passedRatio > CATCH_CONFIG_CONSOLE_WIDTH - 1) - findMax(failedRatio, failedButOkRatio, passedRatio)--; + findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)--; m_stream << m_colour->guardColour( Colour::Error ) << std::string( failedRatio, '=' ) @@ -619,6 +632,8 @@ void ConsoleReporter::printTotalsDivider(Totals const& totals) { m_stream << m_colour->guardColour( Colour::Success ) << std::string( passedRatio, '=' ); } + m_stream << m_colour->guardColour( Colour::Skip ) + << std::string( skippedRatio, '=' ); } else { m_stream << m_colour->guardColour( Colour::Warning ) << std::string( CATCH_CONFIG_CONSOLE_WIDTH - 1, '=' ); diff --git a/src/catch2/reporters/catch_reporter_helpers.cpp b/src/catch2/reporters/catch_reporter_helpers.cpp index f6aa6fc57b..783edce6f0 100644 --- a/src/catch2/reporters/catch_reporter_helpers.cpp +++ b/src/catch2/reporters/catch_reporter_helpers.cpp @@ -325,6 +325,9 @@ namespace Catch { columns.push_back( SummaryColumn( "failed", Colour::ResultError ) .addRow( totals.testCases.failed ) .addRow( totals.assertions.failed ) ); + columns.push_back( SummaryColumn( "skipped", Colour::Skip ) + .addRow( totals.testCases.skipped ) + .addRow( totals.assertions.skipped ) ); columns.push_back( SummaryColumn( "failed as expected", Colour::ResultExpectedFailure ) .addRow( totals.testCases.failedButOk ) diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp index 837d048986..0f94a3ef11 100644 --- a/src/catch2/reporters/catch_reporter_junit.cpp +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -264,6 +264,7 @@ namespace Catch { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: + case ResultWas::ExplicitSkip: elementName = "internalError"; break; } diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp index 365979f48d..5ba3a32391 100644 --- a/src/catch2/reporters/catch_reporter_sonarqube.cpp +++ b/src/catch2/reporters/catch_reporter_sonarqube.cpp @@ -124,6 +124,7 @@ namespace Catch { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: + case ResultWas::ExplicitSkip: elementName = "internalError"; break; } diff --git a/src/catch2/reporters/catch_reporter_tap.cpp b/src/catch2/reporters/catch_reporter_tap.cpp index 59f8fb8b44..4f031524e3 100644 --- a/src/catch2/reporters/catch_reporter_tap.cpp +++ b/src/catch2/reporters/catch_reporter_tap.cpp @@ -104,6 +104,7 @@ namespace Catch { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: + case ResultWas::ExplicitSkip: printResultType("** internal error **"_sr); break; } diff --git a/src/catch2/reporters/catch_reporter_teamcity.cpp b/src/catch2/reporters/catch_reporter_teamcity.cpp index 1d002c27e9..4e53871db3 100644 --- a/src/catch2/reporters/catch_reporter_teamcity.cpp +++ b/src/catch2/reporters/catch_reporter_teamcity.cpp @@ -89,6 +89,7 @@ namespace Catch { case ResultWas::Ok: case ResultWas::Info: case ResultWas::Warning: + case ResultWas::ExplicitSkip: CATCH_ERROR("Internal error in TeamCity reporter"); // These cases are here to prevent compiler warnings case ResultWas::Unknown: diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b40fbcebd6..464fb683ac 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -116,6 +116,7 @@ set(TEST_SOURCES ${SELF_TEST_DIR}/UsageTests/Generators.tests.cpp ${SELF_TEST_DIR}/UsageTests/Message.tests.cpp ${SELF_TEST_DIR}/UsageTests/Misc.tests.cpp + ${SELF_TEST_DIR}/UsageTests/Skip.tests.cpp ${SELF_TEST_DIR}/UsageTests/ToStringByte.tests.cpp ${SELF_TEST_DIR}/UsageTests/ToStringChrono.tests.cpp ${SELF_TEST_DIR}/UsageTests/ToStringGeneral.tests.cpp diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 332439a786..9df53f88e1 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -309,10 +309,12 @@ Message from section two :test-result: PASS comparisons between const int variables :test-result: PASS comparisons between int variables :test-result: PASS convertToBits +:test-result: XFAIL dynamic skipping works with generators :test-result: PASS empty tags are not allowed :test-result: PASS erfc_inv :test-result: PASS estimate_clock_resolution :test-result: PASS even more nested SECTION tests +:test-result: XFAIL failed assertions before SKIP are still reported :test-result: FAIL first tag loose text artifact :test-result: FAIL has printf @@ -352,9 +354,11 @@ loose text artifact :test-result: PASS run_for_at_least, chronometer :test-result: PASS run_for_at_least, int :test-result: FAIL second tag +:test-result: XFAIL sections can be skipped dynamically at runtime :test-result: FAIL send a single char to INFO :test-result: FAIL sends information to INFO :test-result: PASS shortened hide tags are split apart +:test-result: XFAIL skipped tests can optionally provide a reason :test-result: PASS splitString :test-result: FAIL stacks unscoped info in loops :test-result: PASS startsWith @@ -376,6 +380,7 @@ loose text artifact :test-result: PASS strlen3 :test-result: PASS tables :test-result: PASS tags with dots in later positions are not parsed as hidden +:test-result: XFAIL tests can be skipped dynamically at runtime :test-result: FAIL thrown std::strings are translated :test-result: PASS toString on const wchar_t const pointer returns the string contents :test-result: PASS toString on const wchar_t pointer returns the string contents diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index a7461ae5e4..fee3f4c94c 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -302,10 +302,12 @@ :test-result: PASS comparisons between const int variables :test-result: PASS comparisons between int variables :test-result: PASS convertToBits +:test-result: XFAIL dynamic skipping works with generators :test-result: PASS empty tags are not allowed :test-result: PASS erfc_inv :test-result: PASS estimate_clock_resolution :test-result: PASS even more nested SECTION tests +:test-result: XFAIL failed assertions before SKIP are still reported :test-result: FAIL first tag :test-result: FAIL has printf :test-result: PASS is_unary_function @@ -344,9 +346,11 @@ :test-result: PASS run_for_at_least, chronometer :test-result: PASS run_for_at_least, int :test-result: FAIL second tag +:test-result: XFAIL sections can be skipped dynamically at runtime :test-result: FAIL send a single char to INFO :test-result: FAIL sends information to INFO :test-result: PASS shortened hide tags are split apart +:test-result: XFAIL skipped tests can optionally provide a reason :test-result: PASS splitString :test-result: FAIL stacks unscoped info in loops :test-result: PASS startsWith @@ -368,6 +372,7 @@ :test-result: PASS strlen3 :test-result: PASS tables :test-result: PASS tags with dots in later positions are not parsed as hidden +:test-result: XFAIL tests can be skipped dynamically at runtime :test-result: FAIL thrown std::strings are translated :test-result: PASS toString on const wchar_t const pointer returns the string contents :test-result: PASS toString on const wchar_t pointer returns the string contents diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index ff6b287539..c82ab49f6a 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -2149,6 +2149,9 @@ FloatingPoint.tests.cpp:: passed: convertToBits( -0. ) == ( 1ULL << 9223372036854775808 (0x) FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: Tag.tests.cpp:: passed: Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.103560) == Approx(-0.09203687623843015) for: -0.0920368762 == Approx( -0.0920368762 ) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.067400) == Approx(-0.05980291115763361) for: -0.0598029112 == Approx( -0.0598029112 ) @@ -2158,6 +2161,8 @@ InternalBenchmark.tests.cpp:: passed: res.outliers.total() == 0 for Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: +Skip.tests.cpp:: failed: 3 == 4 +Skip.tests.cpp:: ** internal error **: loose text artifact Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -2299,9 +2304,12 @@ InternalBenchmark.tests.cpp:: passed: x >= old_x for: 128 >= 64 InternalBenchmark.tests.cpp:: passed: Timing.elapsed >= time for: 128 ns >= 100 ns InternalBenchmark.tests.cpp:: passed: Timing.result == Timing.iterations + 17 for: 145 == 145 InternalBenchmark.tests.cpp:: passed: Timing.iterations >= time.count() for: 128 >= 100 +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: Misc.tests.cpp:: failed: false with 1 message: '3' Message.tests.cpp:: failed: false with 2 messages: 'hi' and 'i := 7' Tag.tests.cpp:: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +Skip.tests.cpp:: ** internal error **: StringManip.tests.cpp:: passed: splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } StringManip.tests.cpp:: passed: splitStringRef("abc", ','), Equals(std::vector{"abc"}) for: { abc } Equals: { abc } StringManip.tests.cpp:: passed: splitStringRef("abc,def", ','), Equals(std::vector{"abc", "def"}) for: { abc, def } Equals: { abc, def } @@ -2367,6 +2375,7 @@ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_cast(std::get<1>(data)) for: 6 == 6 Tag.tests.cpp:: passed: testcase.tags.size() == 1 for: 1 == 1 Tag.tests.cpp:: passed: testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag +Skip.tests.cpp:: ** internal error **: Exception.tests.cpp:: failed: unexpected exception with message: 'Why would you throw a std::string?' Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" @@ -2463,7 +2472,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 395 | 305 passed | 83 failed | 7 failed as expected -assertions: 2163 | 1993 passed | 143 failed | 27 failed as expected +test cases: 400 | 305 passed | 83 failed | 4 skipped | 8 failed as expected +assertions: 2172 | 1995 passed | 143 failed | 6 skipped | 28 failed as expected diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index d11bdbaef3..fd0499ef99 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -2142,6 +2142,9 @@ FloatingPoint.tests.cpp:: passed: convertToBits( -0. ) == ( 1ULL << 9223372036854775808 (0x) FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: Tag.tests.cpp:: passed: Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.103560) == Approx(-0.09203687623843015) for: -0.0920368762 == Approx( -0.0920368762 ) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.067400) == Approx(-0.05980291115763361) for: -0.0598029112 == Approx( -0.0598029112 ) @@ -2151,6 +2154,8 @@ InternalBenchmark.tests.cpp:: passed: res.outliers.total() == 0 for Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: +Skip.tests.cpp:: failed: 3 == 4 +Skip.tests.cpp:: ** internal error **: Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -2291,9 +2296,12 @@ InternalBenchmark.tests.cpp:: passed: x >= old_x for: 128 >= 64 InternalBenchmark.tests.cpp:: passed: Timing.elapsed >= time for: 128 ns >= 100 ns InternalBenchmark.tests.cpp:: passed: Timing.result == Timing.iterations + 17 for: 145 == 145 InternalBenchmark.tests.cpp:: passed: Timing.iterations >= time.count() for: 128 >= 100 +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: Misc.tests.cpp:: failed: false with 1 message: '3' Message.tests.cpp:: failed: false with 2 messages: 'hi' and 'i := 7' Tag.tests.cpp:: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +Skip.tests.cpp:: ** internal error **: StringManip.tests.cpp:: passed: splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } StringManip.tests.cpp:: passed: splitStringRef("abc", ','), Equals(std::vector{"abc"}) for: { abc } Equals: { abc } StringManip.tests.cpp:: passed: splitStringRef("abc,def", ','), Equals(std::vector{"abc", "def"}) for: { abc, def } Equals: { abc, def } @@ -2359,6 +2367,7 @@ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_cast(std::get<1>(data)) for: 6 == 6 Tag.tests.cpp:: passed: testcase.tags.size() == 1 for: 1 == 1 Tag.tests.cpp:: passed: testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag +Skip.tests.cpp:: ** internal error **: Exception.tests.cpp:: failed: unexpected exception with message: 'Why would you throw a std::string?' Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" @@ -2455,7 +2464,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 395 | 305 passed | 83 failed | 7 failed as expected -assertions: 2163 | 1993 passed | 143 failed | 27 failed as expected +test cases: 400 | 305 passed | 83 failed | 4 skipped | 8 failed as expected +assertions: 2172 | 1995 passed | 143 failed | 6 skipped | 28 failed as expected diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index 39fd845b2c..551e3c825e 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1186,6 +1186,37 @@ Misc.tests.cpp:: FAILED: with expansion: false +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 41 + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + +------------------------------------------------------------------------------- +failed assertions before SKIP are still reported +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + CHECK( 3 == 4 ) + +Skip.tests.cpp:: SKIPPED: + loose text artifact ------------------------------------------------------------------------------- just failure @@ -1338,6 +1369,15 @@ Message.tests.cpp:: FAILED: with message: this SHOULD be seen only ONCE +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- send a single char to INFO ------------------------------------------------------------------------------- @@ -1361,6 +1401,16 @@ with messages: hi i := 7 +------------------------------------------------------------------------------- +skipped tests can optionally provide a reason +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- stacks unscoped info in loops ------------------------------------------------------------------------------- @@ -1383,6 +1433,14 @@ with messages: 5 6 +------------------------------------------------------------------------------- +tests can be skipped dynamically at runtime +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- thrown std::strings are translated ------------------------------------------------------------------------------- @@ -1394,6 +1452,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 395 | 319 passed | 69 failed | 7 failed as expected -assertions: 2148 | 1993 passed | 128 failed | 27 failed as expected +test cases: 400 | 319 passed | 69 failed | 4 skipped | 8 failed as expected +assertions: 2157 | 1995 passed | 128 failed | 6 skipped | 28 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 57fa623954..a1b55c4465 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -15204,6 +15204,34 @@ FloatingPoint.tests.cpp:: PASSED: with expansion: 1 == 1 +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 41 + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- empty tags are not allowed ------------------------------------------------------------------------------- @@ -15279,6 +15307,17 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: +------------------------------------------------------------------------------- +failed assertions before SKIP are still reported +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + CHECK( 3 == 4 ) + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- first tag ------------------------------------------------------------------------------- @@ -16376,6 +16415,24 @@ Misc.tests.cpp: No assertions in test case 'second tag' +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + not skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- send a single char to INFO ------------------------------------------------------------------------------- @@ -16410,6 +16467,16 @@ Tag.tests.cpp:: PASSED: with expansion: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +------------------------------------------------------------------------------- +skipped tests can optionally provide a reason +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- splitString ------------------------------------------------------------------------------- @@ -16837,6 +16904,14 @@ Tag.tests.cpp:: PASSED: with expansion: magic.tag == magic.tag +------------------------------------------------------------------------------- +tests can be skipped dynamically at runtime +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- thrown std::strings are translated ------------------------------------------------------------------------------- @@ -17544,6 +17619,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 395 | 305 passed | 83 failed | 7 failed as expected -assertions: 2163 | 1993 passed | 143 failed | 27 failed as expected +test cases: 400 | 305 passed | 83 failed | 4 skipped | 8 failed as expected +assertions: 2172 | 1995 passed | 143 failed | 6 skipped | 28 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 6e5faa9622..1eec0efd50 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -15197,6 +15197,34 @@ FloatingPoint.tests.cpp:: PASSED: with expansion: 1 == 1 +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 41 + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +dynamic skipping works with generators +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- empty tags are not allowed ------------------------------------------------------------------------------- @@ -15272,6 +15300,17 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: +------------------------------------------------------------------------------- +failed assertions before SKIP are still reported +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + CHECK( 3 == 4 ) + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- first tag ------------------------------------------------------------------------------- @@ -16368,6 +16407,24 @@ Misc.tests.cpp: No assertions in test case 'second tag' +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + not skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- send a single char to INFO ------------------------------------------------------------------------------- @@ -16402,6 +16459,16 @@ Tag.tests.cpp:: PASSED: with expansion: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +------------------------------------------------------------------------------- +skipped tests can optionally provide a reason +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + skipping because answer = 43 + ------------------------------------------------------------------------------- splitString ------------------------------------------------------------------------------- @@ -16829,6 +16896,14 @@ Tag.tests.cpp:: PASSED: with expansion: magic.tag == magic.tag +------------------------------------------------------------------------------- +tests can be skipped dynamically at runtime +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- thrown std::strings are translated ------------------------------------------------------------------------------- @@ -17536,6 +17611,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 395 | 305 passed | 83 failed | 7 failed as expected -assertions: 2163 | 1993 passed | 143 failed | 27 failed as expected +test cases: 400 | 305 passed | 83 failed | 4 skipped | 8 failed as expected +assertions: 2172 | 1995 passed | 143 failed | 6 skipped | 28 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index 795c817115..15fdad0903 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1625,12 +1625,21 @@ at Misc.tests.cpp: + + + + +FAILED: + CHECK( 3 == 4 ) +at Skip.tests.cpp: + + @@ -1794,6 +1803,8 @@ at Message.tests.cpp: + + FAILED: @@ -1812,6 +1823,7 @@ at Message.tests.cpp: + @@ -1856,6 +1868,7 @@ at Message.tests.cpp: + FAILED: diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index 6bacb08883..2b9ce03e76 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -1624,12 +1624,21 @@ at Misc.tests.cpp: + + + + +FAILED: + CHECK( 3 == 4 ) +at Skip.tests.cpp: + + @@ -1793,6 +1802,8 @@ at Message.tests.cpp: + + FAILED: @@ -1811,6 +1822,7 @@ at Message.tests.cpp: + @@ -1855,6 +1867,7 @@ at Message.tests.cpp: + FAILED: diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 16462bf8dd..47e5b873ad 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -1832,6 +1832,20 @@ at Misc.tests.cpp: + + + + +FAILED: + CHECK( 3 == 4 ) +at Skip.tests.cpp: + + + + + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 3f9489bcfc..7c0792d8e6 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -1831,6 +1831,20 @@ at Misc.tests.cpp: + + + + +FAILED: + CHECK( 3 == 4 ) +at Skip.tests.cpp: + + + + + + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 595c1ae751..8a0ea50bbe 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -3779,6 +3779,12 @@ ok {test-number} - convertToBits( -0. ) == ( 1ULL << 63 ) for: 92233720368547758 ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 # convertToBits ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 +# dynamic skipping works with generators +** internal error ** 1886 - +# dynamic skipping works with generators +ok {test-number} - +# dynamic skipping works with generators +** internal error ** 1888 - # empty tags are not allowed ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) # erfc_inv @@ -3797,6 +3803,10 @@ ok {test-number} - ok {test-number} - # even more nested SECTION tests ok {test-number} - +# failed assertions before SKIP are still reported +not ok {test-number} - 3 == 4 +# failed assertions before SKIP are still reported +** internal error ** 1899 - loose text artifact # is_unary_function ok {test-number} - with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -4070,12 +4080,18 @@ ok {test-number} - Timing.elapsed >= time for: 128 ns >= 100 ns ok {test-number} - Timing.result == Timing.iterations + 17 for: 145 == 145 # run_for_at_least, int ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100 +# sections can be skipped dynamically at runtime +ok {test-number} - +# sections can be skipped dynamically at runtime +** internal error ** 2037 - # send a single char to INFO not ok {test-number} - false with 1 message: '3' # sends information to INFO not ok {test-number} - false with 2 messages: 'hi' and 'i := 7' # shortened hide tags are split apart ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +# skipped tests can optionally provide a reason +** internal error ** 2041 - # splitString ok {test-number} - splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } # splitString @@ -4158,6 +4174,8 @@ ok {test-number} - strlen(std::get<0>(data)) == static_cast(std::get<1>( ok {test-number} - testcase.tags.size() == 1 for: 1 == 1 # tags with dots in later positions are not parsed as hidden ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag +# tests can be skipped dynamically at runtime +** internal error ** 2083 - # thrown std::strings are translated not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?' # toString on const wchar_t const pointer returns the string contents @@ -4330,5 +4348,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2163 +1..2172 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index b0fc4c6cae..19fc333355 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -3772,6 +3772,12 @@ ok {test-number} - convertToBits( -0. ) == ( 1ULL << 63 ) for: 92233720368547758 ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 # convertToBits ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 +# dynamic skipping works with generators +** internal error ** 1886 - +# dynamic skipping works with generators +ok {test-number} - +# dynamic skipping works with generators +** internal error ** 1888 - # empty tags are not allowed ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) # erfc_inv @@ -3790,6 +3796,10 @@ ok {test-number} - ok {test-number} - # even more nested SECTION tests ok {test-number} - +# failed assertions before SKIP are still reported +not ok {test-number} - 3 == 4 +# failed assertions before SKIP are still reported +** internal error ** 1899 - # is_unary_function ok {test-number} - with 1 message: 'Catch::Clara::Detail::is_unary_function::value' # is_unary_function @@ -4062,12 +4072,18 @@ ok {test-number} - Timing.elapsed >= time for: 128 ns >= 100 ns ok {test-number} - Timing.result == Timing.iterations + 17 for: 145 == 145 # run_for_at_least, int ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100 +# sections can be skipped dynamically at runtime +ok {test-number} - +# sections can be skipped dynamically at runtime +** internal error ** 2037 - # send a single char to INFO not ok {test-number} - false with 1 message: '3' # sends information to INFO not ok {test-number} - false with 2 messages: 'hi' and 'i := 7' # shortened hide tags are split apart ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) +# skipped tests can optionally provide a reason +** internal error ** 2041 - # splitString ok {test-number} - splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } # splitString @@ -4150,6 +4166,8 @@ ok {test-number} - strlen(std::get<0>(data)) == static_cast(std::get<1>( ok {test-number} - testcase.tags.size() == 1 for: 1 == 1 # tags with dots in later positions are not parsed as hidden ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag +# tests can be skipped dynamically at runtime +** internal error ** 2083 - # thrown std::strings are translated not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?' # toString on const wchar_t const pointer returns the string contents @@ -4322,5 +4340,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2163 +1..2172 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index f0010e2ea3..e793caba04 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -748,6 +748,8 @@ ##teamcity[testFinished name='comparisons between int variables' duration="{duration}"] ##teamcity[testStarted name='convertToBits'] ##teamcity[testFinished name='convertToBits' duration="{duration}"] +##teamcity[testStarted name='dynamic skipping works with generators'] +##teamcity[testFinished name='dynamic skipping works with generators' duration="{duration}"] ##teamcity[testStarted name='empty tags are not allowed'] ##teamcity[testFinished name='empty tags are not allowed' duration="{duration}"] ##teamcity[testStarted name='erfc_inv'] @@ -756,6 +758,9 @@ ##teamcity[testFinished name='estimate_clock_resolution' duration="{duration}"] ##teamcity[testStarted name='even more nested SECTION tests'] ##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"] +##teamcity[testStarted name='failed assertions before SKIP are still reported'] +##teamcity[testIgnored name='failed assertions before SKIP are still reported' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failed assertions before SKIP are still reported' duration="{duration}"] ##teamcity[testStarted name='first tag'] ##teamcity[testFinished name='first tag' duration="{duration}"] ##teamcity[testStarted name='has printf'] @@ -847,6 +852,8 @@ loose text artifact ##teamcity[testFinished name='run_for_at_least, int' duration="{duration}"] ##teamcity[testStarted name='second tag'] ##teamcity[testFinished name='second tag' duration="{duration}"] +##teamcity[testStarted name='sections can be skipped dynamically at runtime'] +##teamcity[testFinished name='sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='send a single char to INFO'] ##teamcity[testFailed name='send a single char to INFO' message='Misc.tests.cpp:|n...............................................................................|n|nMisc.tests.cpp:|nexpression failed with message:|n "3"|n REQUIRE( false )|nwith expansion:|n false|n'] ##teamcity[testFinished name='send a single char to INFO' duration="{duration}"] @@ -855,6 +862,8 @@ loose text artifact ##teamcity[testFinished name='sends information to INFO' duration="{duration}"] ##teamcity[testStarted name='shortened hide tags are split apart'] ##teamcity[testFinished name='shortened hide tags are split apart' duration="{duration}"] +##teamcity[testStarted name='skipped tests can optionally provide a reason'] +##teamcity[testFinished name='skipped tests can optionally provide a reason' duration="{duration}"] ##teamcity[testStarted name='splitString'] ##teamcity[testFinished name='splitString' duration="{duration}"] ##teamcity[testStarted name='stacks unscoped info in loops'] @@ -899,6 +908,8 @@ loose text artifact ##teamcity[testFinished name='tables' duration="{duration}"] ##teamcity[testStarted name='tags with dots in later positions are not parsed as hidden'] ##teamcity[testFinished name='tags with dots in later positions are not parsed as hidden' duration="{duration}"] +##teamcity[testStarted name='tests can be skipped dynamically at runtime'] +##teamcity[testFinished name='tests can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='thrown std::strings are translated'] ##teamcity[testFailed name='thrown std::strings are translated' message='Exception.tests.cpp:|n...............................................................................|n|nException.tests.cpp:|nunexpected exception with message:|n "Why would you throw a std::string?"'] ##teamcity[testFinished name='thrown std::strings are translated' duration="{duration}"] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index 52622a89b0..e949268c05 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -748,6 +748,8 @@ ##teamcity[testFinished name='comparisons between int variables' duration="{duration}"] ##teamcity[testStarted name='convertToBits'] ##teamcity[testFinished name='convertToBits' duration="{duration}"] +##teamcity[testStarted name='dynamic skipping works with generators'] +##teamcity[testFinished name='dynamic skipping works with generators' duration="{duration}"] ##teamcity[testStarted name='empty tags are not allowed'] ##teamcity[testFinished name='empty tags are not allowed' duration="{duration}"] ##teamcity[testStarted name='erfc_inv'] @@ -756,6 +758,9 @@ ##teamcity[testFinished name='estimate_clock_resolution' duration="{duration}"] ##teamcity[testStarted name='even more nested SECTION tests'] ##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"] +##teamcity[testStarted name='failed assertions before SKIP are still reported'] +##teamcity[testIgnored name='failed assertions before SKIP are still reported' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failed assertions before SKIP are still reported' duration="{duration}"] ##teamcity[testStarted name='first tag'] ##teamcity[testFinished name='first tag' duration="{duration}"] ##teamcity[testStarted name='has printf'] @@ -846,6 +851,8 @@ ##teamcity[testFinished name='run_for_at_least, int' duration="{duration}"] ##teamcity[testStarted name='second tag'] ##teamcity[testFinished name='second tag' duration="{duration}"] +##teamcity[testStarted name='sections can be skipped dynamically at runtime'] +##teamcity[testFinished name='sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='send a single char to INFO'] ##teamcity[testFailed name='send a single char to INFO' message='Misc.tests.cpp:|n...............................................................................|n|nMisc.tests.cpp:|nexpression failed with message:|n "3"|n REQUIRE( false )|nwith expansion:|n false|n'] ##teamcity[testFinished name='send a single char to INFO' duration="{duration}"] @@ -854,6 +861,8 @@ ##teamcity[testFinished name='sends information to INFO' duration="{duration}"] ##teamcity[testStarted name='shortened hide tags are split apart'] ##teamcity[testFinished name='shortened hide tags are split apart' duration="{duration}"] +##teamcity[testStarted name='skipped tests can optionally provide a reason'] +##teamcity[testFinished name='skipped tests can optionally provide a reason' duration="{duration}"] ##teamcity[testStarted name='splitString'] ##teamcity[testFinished name='splitString' duration="{duration}"] ##teamcity[testStarted name='stacks unscoped info in loops'] @@ -898,6 +907,8 @@ ##teamcity[testFinished name='tables' duration="{duration}"] ##teamcity[testStarted name='tags with dots in later positions are not parsed as hidden'] ##teamcity[testFinished name='tags with dots in later positions are not parsed as hidden' duration="{duration}"] +##teamcity[testStarted name='tests can be skipped dynamically at runtime'] +##teamcity[testFinished name='tests can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='thrown std::strings are translated'] ##teamcity[testFailed name='thrown std::strings are translated' message='Exception.tests.cpp:|n...............................................................................|n|nException.tests.cpp:|nunexpected exception with message:|n "Why would you throw a std::string?"'] ##teamcity[testFinished name='thrown std::strings are translated' duration="{duration}"] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index cb49412e9c..417900ab78 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -17902,6 +17902,9 @@ There is no extra whitespace here + + + @@ -17977,6 +17980,17 @@ There is no extra whitespace here + + + + 3 == 4 + + + 3 == 4 + + + + @@ -19219,6 +19233,15 @@ loose text artifact + +
+ +
+
+ +
+ + 3 @@ -19261,6 +19284,9 @@ loose text artifact + + + @@ -19718,6 +19744,9 @@ loose text artifact + + + Why would you throw a std::string? @@ -20518,6 +20547,6 @@ loose text artifact - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index b614664626..7a84fc9aa8 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -17902,6 +17902,9 @@ There is no extra whitespace here + + + @@ -17977,6 +17980,17 @@ There is no extra whitespace here + + + + 3 == 4 + + + 3 == 4 + + + + @@ -19218,6 +19232,15 @@ There is no extra whitespace here + +
+ +
+
+ +
+ + 3 @@ -19260,6 +19283,9 @@ There is no extra whitespace here + + + @@ -19717,6 +19743,9 @@ There is no extra whitespace here + + + Why would you throw a std::string? @@ -20517,6 +20546,6 @@ There is no extra whitespace here - - + + diff --git a/tests/SelfTest/UsageTests/Skip.tests.cpp b/tests/SelfTest/UsageTests/Skip.tests.cpp new file mode 100644 index 0000000000..6a47493fad --- /dev/null +++ b/tests/SelfTest/UsageTests/Skip.tests.cpp @@ -0,0 +1,38 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include +#include + +TEST_CASE( "tests can be skipped dynamically at runtime", "[skipping]" ) { + SKIP(); + FAIL( "this is not reached" ); +} + +TEST_CASE( "skipped tests can optionally provide a reason", "[skipping]" ) { + const int answer = 43; + SKIP( "skipping because answer = " << answer ); + FAIL( "this is not reached" ); +} + +TEST_CASE( "sections can be skipped dynamically at runtime", "[skipping]" ) { + SECTION( "not skipped" ) { SUCCEED(); } + SECTION( "skipped" ) { SKIP(); } +} + +TEST_CASE( "dynamic skipping works with generators", "[skipping]" ) { + const int answer = GENERATE( 41, 42, 43 ); + if ( answer != 42 ) { SKIP( "skipping because answer = " << answer ); } + SUCCEED(); +} + +TEST_CASE( "failed assertions before SKIP are still reported", + "[skipping][!shouldfail]" ) { + CHECK( 3 == 4 ); + SKIP(); +} From 2494e5d056140740995bf6e37cf50d3ccd5be907 Mon Sep 17 00:00:00 2001 From: Philip Salzmann Date: Wed, 13 Apr 2022 14:03:35 +0200 Subject: [PATCH 2/9] Don't show "skipped assertions" in console/compact reporters Also extend skip tests to cover a few more use cases. --- .../reporters/catch_reporter_helpers.cpp | 8 +- .../Baselines/automake.sw.approved.txt | 9 +- .../Baselines/automake.sw.multi.approved.txt | 6 +- .../Baselines/compact.sw.approved.txt | 17 ++- .../Baselines/compact.sw.multi.approved.txt | 14 ++- .../Baselines/console.std.approved.txt | 77 ++++++++++++- .../Baselines/console.sw.approved.txt | 109 +++++++++++++++++- .../Baselines/console.sw.multi.approved.txt | 106 ++++++++++++++++- .../Baselines/default.sw.multi.approved.txt | 3 + .../SelfTest/Baselines/junit.sw.approved.txt | 35 +++++- .../Baselines/junit.sw.multi.approved.txt | 35 +++++- .../Baselines/sonarqube.sw.approved.txt | 23 +++- .../Baselines/sonarqube.sw.multi.approved.txt | 23 +++- tests/SelfTest/Baselines/tap.sw.approved.txt | 41 +++++-- .../Baselines/tap.sw.multi.approved.txt | 38 ++++-- .../Baselines/teamcity.sw.approved.txt | 18 ++- .../Baselines/teamcity.sw.multi.approved.txt | 18 ++- tests/SelfTest/Baselines/xml.sw.approved.txt | 54 ++++++++- .../Baselines/xml.sw.multi.approved.txt | 54 ++++++++- tests/SelfTest/UsageTests/Skip.tests.cpp | 37 +++++- 20 files changed, 671 insertions(+), 54 deletions(-) diff --git a/src/catch2/reporters/catch_reporter_helpers.cpp b/src/catch2/reporters/catch_reporter_helpers.cpp index 783edce6f0..ffb32ffb0b 100644 --- a/src/catch2/reporters/catch_reporter_helpers.cpp +++ b/src/catch2/reporters/catch_reporter_helpers.cpp @@ -316,9 +316,12 @@ namespace Catch { } std::vector columns; + // Don't include "skipped assertions" in total count + const auto totalAssertionCount = + totals.assertions.total() - totals.assertions.skipped; columns.push_back( SummaryColumn( "", Colour::None ) .addRow( totals.testCases.total() ) - .addRow( totals.assertions.total() ) ); + .addRow( totalAssertionCount ) ); columns.push_back( SummaryColumn( "passed", Colour::Success ) .addRow( totals.testCases.passed ) .addRow( totals.assertions.passed ) ); @@ -327,7 +330,8 @@ namespace Catch { .addRow( totals.assertions.failed ) ); columns.push_back( SummaryColumn( "skipped", Colour::Skip ) .addRow( totals.testCases.skipped ) - .addRow( totals.assertions.skipped ) ); + // Don't print "skipped assertions" + .addRow( 0 ) ); columns.push_back( SummaryColumn( "failed as expected", Colour::ResultExpectedFailure ) .addRow( totals.testCases.failedButOk ) diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 9df53f88e1..595b5a5273 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -297,6 +297,7 @@ Message from section two :test-result: PASS X/level/1/b :test-result: PASS XmlEncode :test-result: PASS XmlWriter writes boolean attributes as true/false +:test-result: XFAIL a succeeding test can still be skipped :test-result: PASS analyse no analysis :test-result: PASS array -> toString :test-result: PASS benchmark function call @@ -314,7 +315,9 @@ Message from section two :test-result: PASS erfc_inv :test-result: PASS estimate_clock_resolution :test-result: PASS even more nested SECTION tests -:test-result: XFAIL failed assertions before SKIP are still reported +:test-result: XFAIL failed assertions before SKIP cause test case to fail +:test-result: XFAIL failing for some generator values causes entire test case to fail +:test-result: XFAIL failing in some unskipped sections causes entire test case to fail :test-result: FAIL first tag loose text artifact :test-result: FAIL has printf @@ -333,6 +336,10 @@ loose text artifact :test-result: FAIL mix info, unscoped info and warning :test-result: FAIL more nested SECTION tests :test-result: PASS nested SECTION tests +a! +b1! +! +:test-result: FAIL nested sections can be skipped dynamically at runtime :test-result: PASS non streamable - with conv. op :test-result: PASS non-copyable objects :test-result: PASS normal_cdf diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index fee3f4c94c..023d4b4e50 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -290,6 +290,7 @@ :test-result: PASS X/level/1/b :test-result: PASS XmlEncode :test-result: PASS XmlWriter writes boolean attributes as true/false +:test-result: XFAIL a succeeding test can still be skipped :test-result: PASS analyse no analysis :test-result: PASS array -> toString :test-result: PASS benchmark function call @@ -307,7 +308,9 @@ :test-result: PASS erfc_inv :test-result: PASS estimate_clock_resolution :test-result: PASS even more nested SECTION tests -:test-result: XFAIL failed assertions before SKIP are still reported +:test-result: XFAIL failed assertions before SKIP cause test case to fail +:test-result: XFAIL failing for some generator values causes entire test case to fail +:test-result: XFAIL failing in some unskipped sections causes entire test case to fail :test-result: FAIL first tag :test-result: FAIL has printf :test-result: PASS is_unary_function @@ -325,6 +328,7 @@ :test-result: FAIL mix info, unscoped info and warning :test-result: FAIL more nested SECTION tests :test-result: PASS nested SECTION tests +:test-result: FAIL nested sections can be skipped dynamically at runtime :test-result: PASS non streamable - with conv. op :test-result: PASS non-copyable objects :test-result: PASS normal_cdf diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index c82ab49f6a..bb78e29b5f 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -2060,6 +2060,8 @@ Xml.tests.cpp:: passed: encode( "[\x7F]" ) == "[\\x7F]" for: "[\x7F Xml.tests.cpp:: passed: stream.str(), ContainsSubstring(R"(attr1="true")") && ContainsSubstring(R"(attr2="false")") for: " " ( contains: "attr1="true"" and contains: "attr2="false"" ) +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: InternalBenchmark.tests.cpp:: passed: analysis.mean.point.count() == 23 for: 23.0 == 23 InternalBenchmark.tests.cpp:: passed: analysis.mean.lower_bound.count() == 23 for: 23.0 == 23 InternalBenchmark.tests.cpp:: passed: analysis.mean.upper_bound.count() == 23 for: 23.0 == 23 @@ -2163,6 +2165,12 @@ Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Skip.tests.cpp:: failed: 3 == 4 Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: failed: explicitly +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: failed: explicitly +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: failed: explicitly loose text artifact Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -2220,6 +2228,10 @@ Misc.tests.cpp:: passed: a < b for: 1 < 2 Misc.tests.cpp:: passed: a != b for: 1 != 2 Misc.tests.cpp:: passed: b != a for: 2 != 1 Misc.tests.cpp:: passed: a != b for: 1 != 2 +a! +b1! +Skip.tests.cpp:: ** internal error **: +! Tricky.tests.cpp:: passed: s == "7" for: "7" == "7" Tricky.tests.cpp:: passed: ti == typeid(int) for: {?} == {?} InternalBenchmark.tests.cpp:: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 ) @@ -2306,6 +2318,7 @@ InternalBenchmark.tests.cpp:: passed: Timing.result == Timing.itera InternalBenchmark.tests.cpp:: passed: Timing.iterations >= time.count() for: 128 >= 100 Skip.tests.cpp:: passed: Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: passed: Misc.tests.cpp:: failed: false with 1 message: '3' Message.tests.cpp:: failed: false with 2 messages: 'hi' and 'i := 7' Tag.tests.cpp:: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) @@ -2472,7 +2485,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 400 | 305 passed | 83 failed | 4 skipped | 8 failed as expected -assertions: 2172 | 1995 passed | 143 failed | 6 skipped | 28 failed as expected +test cases: 404 | 305 passed | 84 failed | 5 skipped | 10 failed as expected +assertions: 2173 | 1997 passed | 145 failed | 31 failed as expected diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index fd0499ef99..8d0c48291f 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -2053,6 +2053,8 @@ Xml.tests.cpp:: passed: encode( "[\x7F]" ) == "[\\x7F]" for: "[\x7F Xml.tests.cpp:: passed: stream.str(), ContainsSubstring(R"(attr1="true")") && ContainsSubstring(R"(attr2="false")") for: " " ( contains: "attr1="true"" and contains: "attr2="false"" ) +Skip.tests.cpp:: passed: +Skip.tests.cpp:: ** internal error **: InternalBenchmark.tests.cpp:: passed: analysis.mean.point.count() == 23 for: 23.0 == 23 InternalBenchmark.tests.cpp:: passed: analysis.mean.lower_bound.count() == 23 for: 23.0 == 23 InternalBenchmark.tests.cpp:: passed: analysis.mean.upper_bound.count() == 23 for: 23.0 == 23 @@ -2156,6 +2158,12 @@ Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Skip.tests.cpp:: failed: 3 == 4 Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: failed: explicitly +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: failed: explicitly +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: failed: explicitly Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -2212,6 +2220,7 @@ Misc.tests.cpp:: passed: a < b for: 1 < 2 Misc.tests.cpp:: passed: a != b for: 1 != 2 Misc.tests.cpp:: passed: b != a for: 2 != 1 Misc.tests.cpp:: passed: a != b for: 1 != 2 +Skip.tests.cpp:: ** internal error **: Tricky.tests.cpp:: passed: s == "7" for: "7" == "7" Tricky.tests.cpp:: passed: ti == typeid(int) for: {?} == {?} InternalBenchmark.tests.cpp:: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 ) @@ -2298,6 +2307,7 @@ InternalBenchmark.tests.cpp:: passed: Timing.result == Timing.itera InternalBenchmark.tests.cpp:: passed: Timing.iterations >= time.count() for: 128 >= 100 Skip.tests.cpp:: passed: Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: passed: Misc.tests.cpp:: failed: false with 1 message: '3' Message.tests.cpp:: failed: false with 2 messages: 'hi' and 'i := 7' Tag.tests.cpp:: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) @@ -2464,7 +2474,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 400 | 305 passed | 83 failed | 4 skipped | 8 failed as expected -assertions: 2172 | 1995 passed | 143 failed | 6 skipped | 28 failed as expected +test cases: 404 | 305 passed | 84 failed | 5 skipped | 10 failed as expected +assertions: 2173 | 1997 passed | 145 failed | 31 failed as expected diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index 551e3c825e..dd3199f04c 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1164,6 +1164,14 @@ Exception.tests.cpp:: FAILED: due to unexpected exception with message: unexpected exception +------------------------------------------------------------------------------- +a succeeding test can still be skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- checkedElse, failing ------------------------------------------------------------------------------- @@ -1207,7 +1215,7 @@ explicitly with message: skipping because answer = 43 ------------------------------------------------------------------------------- -failed assertions before SKIP are still reported +failed assertions before SKIP cause test case to fail ------------------------------------------------------------------------------- Skip.tests.cpp: ............................................................................... @@ -1217,6 +1225,56 @@ Skip.tests.cpp:: FAILED: Skip.tests.cpp:: SKIPPED: +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +------------------------------------------------------------------------------- +failing in some unskipped sections causes entire test case to fail + skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +------------------------------------------------------------------------------- +failing in some unskipped sections causes entire test case to fail + not skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + loose text artifact ------------------------------------------------------------------------------- just failure @@ -1335,6 +1393,19 @@ Misc.tests.cpp:: FAILED: with expansion: 1 == 2 +a! +b1! +------------------------------------------------------------------------------- +nested sections can be skipped dynamically at runtime + B + B2 +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +! ------------------------------------------------------------------------------- not prints unscoped info from previous failures ------------------------------------------------------------------------------- @@ -1452,6 +1523,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 400 | 319 passed | 69 failed | 4 skipped | 8 failed as expected -assertions: 2157 | 1995 passed | 128 failed | 6 skipped | 28 failed as expected +test cases: 404 | 319 passed | 69 failed | 6 skipped | 10 failed as expected +assertions: 2156 | 1997 passed | 128 failed | 31 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index a1b55c4465..48609d97ff 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -14659,6 +14659,16 @@ with expansion: " ( contains: "attr1="true"" and contains: "attr2="false"" ) +------------------------------------------------------------------------------- +a succeeding test can still be skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- analyse no analysis ------------------------------------------------------------------------------- @@ -15308,7 +15318,7 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: ------------------------------------------------------------------------------- -failed assertions before SKIP are still reported +failed assertions before SKIP cause test case to fail ------------------------------------------------------------------------------- Skip.tests.cpp: ............................................................................... @@ -15318,6 +15328,56 @@ Skip.tests.cpp:: FAILED: Skip.tests.cpp:: SKIPPED: +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +------------------------------------------------------------------------------- +failing in some unskipped sections causes entire test case to fail + skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +------------------------------------------------------------------------------- +failing in some unskipped sections causes entire test case to fail + not skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + ------------------------------------------------------------------------------- first tag ------------------------------------------------------------------------------- @@ -15814,6 +15874,40 @@ Misc.tests.cpp:: PASSED: with expansion: 1 != 2 +a------------------------------------------------------------------------------- +nested sections can be skipped dynamically at runtime + A +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + + +No assertions in section 'A' + +! +b1------------------------------------------------------------------------------- +nested sections can be skipped dynamically at runtime + B + B1 +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + + +No assertions in section 'B1' + +! +------------------------------------------------------------------------------- +nested sections can be skipped dynamically at runtime + B + B2 +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +! ------------------------------------------------------------------------------- non streamable - with conv. op ------------------------------------------------------------------------------- @@ -16433,6 +16527,15 @@ Skip.tests.cpp: Skip.tests.cpp:: SKIPPED: +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + also not skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + ------------------------------------------------------------------------------- send a single char to INFO ------------------------------------------------------------------------------- @@ -17619,6 +17722,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 400 | 305 passed | 83 failed | 4 skipped | 8 failed as expected -assertions: 2172 | 1995 passed | 143 failed | 6 skipped | 28 failed as expected +test cases: 404 | 305 passed | 84 failed | 5 skipped | 10 failed as expected +assertions: 2173 | 1997 passed | 145 failed | 31 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 1eec0efd50..aa3194121c 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -14652,6 +14652,16 @@ with expansion: " ( contains: "attr1="true"" and contains: "attr2="false"" ) +------------------------------------------------------------------------------- +a succeeding test can still be skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- analyse no analysis ------------------------------------------------------------------------------- @@ -15301,7 +15311,7 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: ------------------------------------------------------------------------------- -failed assertions before SKIP are still reported +failed assertions before SKIP cause test case to fail ------------------------------------------------------------------------------- Skip.tests.cpp: ............................................................................... @@ -15311,6 +15321,56 @@ Skip.tests.cpp:: FAILED: Skip.tests.cpp:: SKIPPED: +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +failing for some generator values causes entire test case to fail +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +------------------------------------------------------------------------------- +failing in some unskipped sections causes entire test case to fail + skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + +------------------------------------------------------------------------------- +failing in some unskipped sections causes entire test case to fail + not skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: FAILED: + ------------------------------------------------------------------------------- first tag ------------------------------------------------------------------------------- @@ -15806,6 +15866,37 @@ Misc.tests.cpp:: PASSED: with expansion: 1 != 2 +------------------------------------------------------------------------------- +nested sections can be skipped dynamically at runtime + A +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + + +No assertions in section 'A' + +------------------------------------------------------------------------------- +nested sections can be skipped dynamically at runtime + B + B1 +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + + +No assertions in section 'B1' + +------------------------------------------------------------------------------- +nested sections can be skipped dynamically at runtime + B + B2 +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: + ------------------------------------------------------------------------------- non streamable - with conv. op ------------------------------------------------------------------------------- @@ -16425,6 +16516,15 @@ Skip.tests.cpp: Skip.tests.cpp:: SKIPPED: +------------------------------------------------------------------------------- +sections can be skipped dynamically at runtime + also not skipped +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: PASSED: + ------------------------------------------------------------------------------- send a single char to INFO ------------------------------------------------------------------------------- @@ -17611,6 +17711,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 400 | 305 passed | 83 failed | 4 skipped | 8 failed as expected -assertions: 2172 | 1995 passed | 143 failed | 6 skipped | 28 failed as expected +test cases: 404 | 305 passed | 84 failed | 5 skipped | 10 failed as expected +assertions: 2173 | 1997 passed | 145 failed | 31 failed as expected diff --git a/tests/SelfTest/Baselines/default.sw.multi.approved.txt b/tests/SelfTest/Baselines/default.sw.multi.approved.txt index 12717aa5f9..bb17484488 100644 --- a/tests/SelfTest/Baselines/default.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/default.sw.multi.approved.txt @@ -6,3 +6,6 @@ A string sent to stderr via clog Message from section one Message from section two loose text artifact +a! +b1! +! diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index 15fdad0903..28cd5c399d 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1591,6 +1591,7 @@ at Exception.tests.cpp: + @@ -1632,11 +1633,29 @@ at Misc.tests.cpp: - + FAILED: CHECK( 3 == 4 ) +at Skip.tests.cpp: + + + + +FAILED: +at Skip.tests.cpp: + + +FAILED: +at Skip.tests.cpp: + + + + + + +FAILED: at Skip.tests.cpp: @@ -1752,6 +1771,14 @@ at Misc.tests.cpp: + + + +a! +b1! +! + + @@ -1805,6 +1832,7 @@ at Message.tests.cpp: + FAILED: @@ -1918,6 +1946,9 @@ This would not be caught previously A string sent directly to stdout Message from section one Message from section two +a! +b1! +! Nor would this diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index 2b9ce03e76..d143f532d0 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -1590,6 +1590,7 @@ at Exception.tests.cpp: + @@ -1631,11 +1632,29 @@ at Misc.tests.cpp: - + FAILED: CHECK( 3 == 4 ) +at Skip.tests.cpp: + + + + +FAILED: +at Skip.tests.cpp: + + +FAILED: +at Skip.tests.cpp: + + + + + + +FAILED: at Skip.tests.cpp: @@ -1751,6 +1770,14 @@ at Misc.tests.cpp: + + + +a! +b1! +! + + @@ -1804,6 +1831,7 @@ at Message.tests.cpp: + FAILED: @@ -1917,6 +1945,9 @@ This would not be caught previously A string sent directly to stdout Message from section one Message from section two +a! +b1! +! Nor would this diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 47e5b873ad..d83952d82d 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -1833,16 +1833,37 @@ at Misc.tests.cpp: + - + FAILED: CHECK( 3 == 4 ) at Skip.tests.cpp: + + +FAILED: +at Skip.tests.cpp: + + +FAILED: +at Skip.tests.cpp: + + + + + +FAILED: +at Skip.tests.cpp: + + + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 7c0792d8e6..7ac035fb15 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -1832,16 +1832,37 @@ at Misc.tests.cpp: + - + FAILED: CHECK( 3 == 4 ) at Skip.tests.cpp: + + +FAILED: +at Skip.tests.cpp: + + +FAILED: +at Skip.tests.cpp: + + + + + +FAILED: +at Skip.tests.cpp: + + + + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 8a0ea50bbe..cca4616030 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -3609,6 +3609,10 @@ ok {test-number} - encode( "[\x01]" ) == "[\\x01]" for: "[\x01]" == "[\x01]" ok {test-number} - encode( "[\x7F]" ) == "[\\x7F]" for: "[\x7F]" == "[\x7F]" # XmlWriter writes boolean attributes as true/false ok {test-number} - stream.str(), ContainsSubstring(R"(attr1="true")") && ContainsSubstring(R"(attr2="false")") for: " " ( contains: "attr1="true"" and contains: "attr2="false"" ) +# a succeeding test can still be skipped +ok {test-number} - +# a succeeding test can still be skipped +** internal error ** 1802 - # analyse no analysis ok {test-number} - analysis.mean.point.count() == 23 for: 23.0 == 23 # analyse no analysis @@ -3780,11 +3784,11 @@ ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == # convertToBits ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 # dynamic skipping works with generators -** internal error ** 1886 - +** internal error ** 1888 - # dynamic skipping works with generators ok {test-number} - # dynamic skipping works with generators -** internal error ** 1888 - +** internal error ** 1890 - # empty tags are not allowed ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) # erfc_inv @@ -3803,10 +3807,22 @@ ok {test-number} - ok {test-number} - # even more nested SECTION tests ok {test-number} - -# failed assertions before SKIP are still reported +# failed assertions before SKIP cause test case to fail not ok {test-number} - 3 == 4 -# failed assertions before SKIP are still reported -** internal error ** 1899 - +# failed assertions before SKIP cause test case to fail +** internal error ** 1901 - +# failing for some generator values causes entire test case to fail +not ok {test-number} - explicitly +# failing for some generator values causes entire test case to fail +** internal error ** 1903 - +# failing for some generator values causes entire test case to fail +not ok {test-number} - explicitly +# failing for some generator values causes entire test case to fail +** internal error ** 1905 - +# failing in some unskipped sections causes entire test case to fail +** internal error ** 1906 - +# failing in some unskipped sections causes entire test case to fail +not ok {test-number} - explicitly loose text artifact # is_unary_function ok {test-number} - with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -3916,6 +3932,11 @@ ok {test-number} - a != b for: 1 != 2 ok {test-number} - b != a for: 2 != 1 # nested SECTION tests ok {test-number} - a != b for: 1 != 2 +a! +b1! +# nested sections can be skipped dynamically at runtime +** internal error ** 1962 - +! # non streamable - with conv. op ok {test-number} - s == "7" for: "7" == "7" # non-copyable objects @@ -4083,7 +4104,9 @@ ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100 # sections can be skipped dynamically at runtime ok {test-number} - # sections can be skipped dynamically at runtime -** internal error ** 2037 - +** internal error ** 2046 - +# sections can be skipped dynamically at runtime +ok {test-number} - # send a single char to INFO not ok {test-number} - false with 1 message: '3' # sends information to INFO @@ -4091,7 +4114,7 @@ not ok {test-number} - false with 2 messages: 'hi' and 'i := 7' # shortened hide tags are split apart ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) # skipped tests can optionally provide a reason -** internal error ** 2041 - +** internal error ** 2051 - # splitString ok {test-number} - splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } # splitString @@ -4175,7 +4198,7 @@ ok {test-number} - testcase.tags.size() == 1 for: 1 == 1 # tags with dots in later positions are not parsed as hidden ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag # tests can be skipped dynamically at runtime -** internal error ** 2083 - +** internal error ** 2093 - # thrown std::strings are translated not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?' # toString on const wchar_t const pointer returns the string contents @@ -4348,5 +4371,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2172 +1..2184 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 19fc333355..4217c05ff7 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -3602,6 +3602,10 @@ ok {test-number} - encode( "[\x01]" ) == "[\\x01]" for: "[\x01]" == "[\x01]" ok {test-number} - encode( "[\x7F]" ) == "[\\x7F]" for: "[\x7F]" == "[\x7F]" # XmlWriter writes boolean attributes as true/false ok {test-number} - stream.str(), ContainsSubstring(R"(attr1="true")") && ContainsSubstring(R"(attr2="false")") for: " " ( contains: "attr1="true"" and contains: "attr2="false"" ) +# a succeeding test can still be skipped +ok {test-number} - +# a succeeding test can still be skipped +** internal error ** 1802 - # analyse no analysis ok {test-number} - analysis.mean.point.count() == 23 for: 23.0 == 23 # analyse no analysis @@ -3773,11 +3777,11 @@ ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == # convertToBits ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 # dynamic skipping works with generators -** internal error ** 1886 - +** internal error ** 1888 - # dynamic skipping works with generators ok {test-number} - # dynamic skipping works with generators -** internal error ** 1888 - +** internal error ** 1890 - # empty tags are not allowed ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) # erfc_inv @@ -3796,10 +3800,22 @@ ok {test-number} - ok {test-number} - # even more nested SECTION tests ok {test-number} - -# failed assertions before SKIP are still reported +# failed assertions before SKIP cause test case to fail not ok {test-number} - 3 == 4 -# failed assertions before SKIP are still reported -** internal error ** 1899 - +# failed assertions before SKIP cause test case to fail +** internal error ** 1901 - +# failing for some generator values causes entire test case to fail +not ok {test-number} - explicitly +# failing for some generator values causes entire test case to fail +** internal error ** 1903 - +# failing for some generator values causes entire test case to fail +not ok {test-number} - explicitly +# failing for some generator values causes entire test case to fail +** internal error ** 1905 - +# failing in some unskipped sections causes entire test case to fail +** internal error ** 1906 - +# failing in some unskipped sections causes entire test case to fail +not ok {test-number} - explicitly # is_unary_function ok {test-number} - with 1 message: 'Catch::Clara::Detail::is_unary_function::value' # is_unary_function @@ -3908,6 +3924,8 @@ ok {test-number} - a != b for: 1 != 2 ok {test-number} - b != a for: 2 != 1 # nested SECTION tests ok {test-number} - a != b for: 1 != 2 +# nested sections can be skipped dynamically at runtime +** internal error ** 1962 - # non streamable - with conv. op ok {test-number} - s == "7" for: "7" == "7" # non-copyable objects @@ -4075,7 +4093,9 @@ ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100 # sections can be skipped dynamically at runtime ok {test-number} - # sections can be skipped dynamically at runtime -** internal error ** 2037 - +** internal error ** 2046 - +# sections can be skipped dynamically at runtime +ok {test-number} - # send a single char to INFO not ok {test-number} - false with 1 message: '3' # sends information to INFO @@ -4083,7 +4103,7 @@ not ok {test-number} - false with 2 messages: 'hi' and 'i := 7' # shortened hide tags are split apart ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) # skipped tests can optionally provide a reason -** internal error ** 2041 - +** internal error ** 2051 - # splitString ok {test-number} - splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } # splitString @@ -4167,7 +4187,7 @@ ok {test-number} - testcase.tags.size() == 1 for: 1 == 1 # tags with dots in later positions are not parsed as hidden ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag # tests can be skipped dynamically at runtime -** internal error ** 2083 - +** internal error ** 2093 - # thrown std::strings are translated not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?' # toString on const wchar_t const pointer returns the string contents @@ -4340,5 +4360,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2172 +1..2184 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index e793caba04..4c97262046 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -722,6 +722,8 @@ ##teamcity[testFinished name='XmlEncode' duration="{duration}"] ##teamcity[testStarted name='XmlWriter writes boolean attributes as true/false'] ##teamcity[testFinished name='XmlWriter writes boolean attributes as true/false' duration="{duration}"] +##teamcity[testStarted name='a succeeding test can still be skipped'] +##teamcity[testFinished name='a succeeding test can still be skipped' duration="{duration}"] ##teamcity[testStarted name='analyse no analysis'] ##teamcity[testFinished name='analyse no analysis' duration="{duration}"] ##teamcity[testStarted name='array -> toString'] @@ -758,9 +760,16 @@ ##teamcity[testFinished name='estimate_clock_resolution' duration="{duration}"] ##teamcity[testStarted name='even more nested SECTION tests'] ##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"] -##teamcity[testStarted name='failed assertions before SKIP are still reported'] -##teamcity[testIgnored name='failed assertions before SKIP are still reported' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] -##teamcity[testFinished name='failed assertions before SKIP are still reported' duration="{duration}"] +##teamcity[testStarted name='failed assertions before SKIP cause test case to fail'] +##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failed assertions before SKIP cause test case to fail' duration="{duration}"] +##teamcity[testStarted name='failing for some generator values causes entire test case to fail'] +##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failing for some generator values causes entire test case to fail' duration="{duration}"] +##teamcity[testStarted name='failing in some unskipped sections causes entire test case to fail'] +##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nnot skipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failing in some unskipped sections causes entire test case to fail' duration="{duration}"] ##teamcity[testStarted name='first tag'] ##teamcity[testFinished name='first tag' duration="{duration}"] ##teamcity[testStarted name='has printf'] @@ -807,6 +816,9 @@ loose text artifact ##teamcity[testFinished name='more nested SECTION tests' duration="{duration}"] ##teamcity[testStarted name='nested SECTION tests'] ##teamcity[testFinished name='nested SECTION tests' duration="{duration}"] +##teamcity[testStarted name='nested sections can be skipped dynamically at runtime'] +##teamcity[testStdOut name='nested sections can be skipped dynamically at runtime' out='a!|nb1!|n!|n'] +##teamcity[testFinished name='nested sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='non streamable - with conv. op'] ##teamcity[testFinished name='non streamable - with conv. op' duration="{duration}"] ##teamcity[testStarted name='non-copyable objects'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index e949268c05..9a6b61ed3a 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -722,6 +722,8 @@ ##teamcity[testFinished name='XmlEncode' duration="{duration}"] ##teamcity[testStarted name='XmlWriter writes boolean attributes as true/false'] ##teamcity[testFinished name='XmlWriter writes boolean attributes as true/false' duration="{duration}"] +##teamcity[testStarted name='a succeeding test can still be skipped'] +##teamcity[testFinished name='a succeeding test can still be skipped' duration="{duration}"] ##teamcity[testStarted name='analyse no analysis'] ##teamcity[testFinished name='analyse no analysis' duration="{duration}"] ##teamcity[testStarted name='array -> toString'] @@ -758,9 +760,16 @@ ##teamcity[testFinished name='estimate_clock_resolution' duration="{duration}"] ##teamcity[testStarted name='even more nested SECTION tests'] ##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"] -##teamcity[testStarted name='failed assertions before SKIP are still reported'] -##teamcity[testIgnored name='failed assertions before SKIP are still reported' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] -##teamcity[testFinished name='failed assertions before SKIP are still reported' duration="{duration}"] +##teamcity[testStarted name='failed assertions before SKIP cause test case to fail'] +##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failed assertions before SKIP cause test case to fail' duration="{duration}"] +##teamcity[testStarted name='failing for some generator values causes entire test case to fail'] +##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failing for some generator values causes entire test case to fail' duration="{duration}"] +##teamcity[testStarted name='failing in some unskipped sections causes entire test case to fail'] +##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nnot skipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='failing in some unskipped sections causes entire test case to fail' duration="{duration}"] ##teamcity[testStarted name='first tag'] ##teamcity[testFinished name='first tag' duration="{duration}"] ##teamcity[testStarted name='has printf'] @@ -806,6 +815,9 @@ ##teamcity[testFinished name='more nested SECTION tests' duration="{duration}"] ##teamcity[testStarted name='nested SECTION tests'] ##teamcity[testFinished name='nested SECTION tests' duration="{duration}"] +##teamcity[testStarted name='nested sections can be skipped dynamically at runtime'] +##teamcity[testStdOut name='nested sections can be skipped dynamically at runtime' out='a!|nb1!|n!|n'] +##teamcity[testFinished name='nested sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='non streamable - with conv. op'] ##teamcity[testFinished name='non streamable - with conv. op' duration="{duration}"] ##teamcity[testStarted name='non-copyable objects'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 417900ab78..0d4ea7a819 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -17158,6 +17158,9 @@ There is no extra whitespace here + + + @@ -17980,7 +17983,7 @@ There is no extra whitespace here - + 3 == 4 @@ -17991,6 +17994,21 @@ There is no extra whitespace here + + + + + + +
+ +
+
+ + +
+ + @@ -18460,6 +18478,33 @@ loose text artifact + +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+ +
+ + +a! +b1! +! + + + @@ -19240,6 +19285,9 @@ loose text artifact
+
+ +
@@ -20547,6 +20595,6 @@ loose text artifact - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index 7a84fc9aa8..1e54761035 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -17158,6 +17158,9 @@ There is no extra whitespace here + + + @@ -17980,7 +17983,7 @@ There is no extra whitespace here - + 3 == 4 @@ -17991,6 +17994,21 @@ There is no extra whitespace here + + + + + + +
+ +
+
+ + +
+ + @@ -18459,6 +18477,33 @@ There is no extra whitespace here + +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+ +
+ + +a! +b1! +! + + + @@ -19239,6 +19284,9 @@ There is no extra whitespace here
+
+ +
@@ -20546,6 +20594,6 @@ There is no extra whitespace here - - + + diff --git a/tests/SelfTest/UsageTests/Skip.tests.cpp b/tests/SelfTest/UsageTests/Skip.tests.cpp index 6a47493fad..70feafaf44 100644 --- a/tests/SelfTest/UsageTests/Skip.tests.cpp +++ b/tests/SelfTest/UsageTests/Skip.tests.cpp @@ -9,6 +9,8 @@ #include #include +#include + TEST_CASE( "tests can be skipped dynamically at runtime", "[skipping]" ) { SKIP(); FAIL( "this is not reached" ); @@ -23,6 +25,17 @@ TEST_CASE( "skipped tests can optionally provide a reason", "[skipping]" ) { TEST_CASE( "sections can be skipped dynamically at runtime", "[skipping]" ) { SECTION( "not skipped" ) { SUCCEED(); } SECTION( "skipped" ) { SKIP(); } + SECTION( "also not skipped" ) { SUCCEED(); } +} + +TEST_CASE( "nested sections can be skipped dynamically at runtime", + "[skipping]" ) { + SECTION( "A" ) { std::cout << "a"; } + SECTION( "B" ) { + SECTION( "B1" ) { std::cout << "b1"; } + SECTION( "B2" ) { SKIP(); } + } + std::cout << "!\n"; } TEST_CASE( "dynamic skipping works with generators", "[skipping]" ) { @@ -31,8 +44,30 @@ TEST_CASE( "dynamic skipping works with generators", "[skipping]" ) { SUCCEED(); } -TEST_CASE( "failed assertions before SKIP are still reported", +TEST_CASE( "failed assertions before SKIP cause test case to fail", "[skipping][!shouldfail]" ) { CHECK( 3 == 4 ); SKIP(); } + +TEST_CASE( "a succeeding test can still be skipped", + "[skipping][!shouldfail]" ) { + SUCCEED(); + SKIP(); +} + +TEST_CASE( "failing in some unskipped sections causes entire test case to fail", + "[skipping][!shouldfail]" ) { + SECTION( "skipped" ) { SKIP(); } + SECTION( "not skipped" ) { FAIL(); } +} + +TEST_CASE( "failing for some generator values causes entire test case to fail", + "[skipping][!shouldfail]" ) { + int i = GENERATE( 1, 2, 3, 4 ); + if ( i % 2 == 0 ) { + SKIP(); + } else { + FAIL(); + } +} From 98e2a888817343bdcd0c5cf962a1d21de6d20086 Mon Sep 17 00:00:00 2001 From: Philip Salzmann Date: Wed, 13 Apr 2022 14:18:35 +0200 Subject: [PATCH 3/9] Return exit code 4 if all test cases are skipped --- src/catch2/catch_session.cpp | 5 +++++ tests/ExtraTests/CMakeLists.txt | 17 +++++++++++++++++ tests/ExtraTests/X93-AllSkipped.cpp | 16 ++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 tests/ExtraTests/X93-AllSkipped.cpp diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp index 128f21d9ac..c89368815f 100644 --- a/src/catch2/catch_session.cpp +++ b/src/catch2/catch_session.cpp @@ -341,6 +341,11 @@ namespace Catch { return 2; } + if ( totals.testCases.total() > 0 && + totals.testCases.total() == totals.testCases.skipped ) { + return 4; + } + // Note that on unices only the lower 8 bits are usually used, clamping // the return value to 255 prevents false negative when some multiple // of 256 tests has failed diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt index d5b27fbaea..4172d7a034 100644 --- a/tests/ExtraTests/CMakeLists.txt +++ b/tests/ExtraTests/CMakeLists.txt @@ -488,15 +488,32 @@ set_tests_properties(TestSpecs::EmptySpecWithNoTestsFails PROPERTIES WILL_FAIL ON ) + add_test( NAME TestSpecs::OverrideFailureWithEmptySpec COMMAND $ --allow-running-no-tests ) + add_test( NAME List::Listeners::WorksWithoutRegisteredListeners COMMAND $ --list-listeners ) + + +add_executable(AllSkipped ${TESTS_DIR}/X93-AllSkipped.cpp) +target_link_libraries(AllSkipped PRIVATE Catch2::Catch2WithMain) + +add_test( + NAME TestSpecs::SkippingAllTestsFails + COMMAND $ +) +set_tests_properties(TestSpecs::SkippingAllTestsFails + PROPERTIES + WILL_FAIL ON +) + set( EXTRA_TEST_BINARIES + AllSkipped PrefixedMacros DisabledMacros DisabledExceptions-DefaultHandler diff --git a/tests/ExtraTests/X93-AllSkipped.cpp b/tests/ExtraTests/X93-AllSkipped.cpp new file mode 100644 index 0000000000..f298116974 --- /dev/null +++ b/tests/ExtraTests/X93-AllSkipped.cpp @@ -0,0 +1,16 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include + +TEST_CASE( "this test case is being skipped" ) { SKIP(); } + +TEST_CASE( "all sections in this test case are being skipped" ) { + SECTION( "A" ) { SKIP(); } + SECTION( "B" ) { SKIP(); } +} From 393048515aa187c7033f292233b48b2ec5d8c5e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Sun, 2 Oct 2022 11:23:32 +0200 Subject: [PATCH 4/9] Use LightGrey for the skip colour This isn't great, but is better than the deep blue that was borderline invisible on dark backgrounds. The fix is to redo the colouring a bit, including introducing light-blue that is actually visible. --- src/catch2/internal/catch_console_colour.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catch2/internal/catch_console_colour.hpp b/src/catch2/internal/catch_console_colour.hpp index 8ee808cc2c..d914431574 100644 --- a/src/catch2/internal/catch_console_colour.hpp +++ b/src/catch2/internal/catch_console_colour.hpp @@ -47,7 +47,7 @@ namespace Catch { Error = BrightRed, Success = Green, - Skip = Blue, + Skip = LightGrey, OriginalExpression = Cyan, ReconstructedExpression = BrightYellow, From 884bb5c728d41230856ac1ab4bf8bb160e98d56d Mon Sep 17 00:00:00 2001 From: Philip Salzmann Date: Wed, 21 Dec 2022 12:43:00 +0100 Subject: [PATCH 5/9] Add support for explicit skips in all reporters --- .../reporters/catch_reporter_automake.cpp | 4 +- .../reporters/catch_reporter_compact.cpp | 8 +- src/catch2/reporters/catch_reporter_junit.cpp | 15 +- .../reporters/catch_reporter_sonarqube.cpp | 16 +- src/catch2/reporters/catch_reporter_tap.cpp | 7 +- .../reporters/catch_reporter_teamcity.cpp | 23 +- src/catch2/reporters/catch_reporter_xml.cpp | 19 +- .../Baselines/automake.sw.approved.txt | 10 +- .../Baselines/automake.sw.multi.approved.txt | 10 +- .../Baselines/compact.sw.approved.txt | 22 +- .../Baselines/compact.sw.multi.approved.txt | 22 +- .../SelfTest/Baselines/junit.sw.approved.txt | 70 +- .../Baselines/junit.sw.multi.approved.txt | 70 +- .../Baselines/sonarqube.sw.approved.txt | 68 +- .../Baselines/sonarqube.sw.multi.approved.txt | 68 +- tests/SelfTest/Baselines/tap.sw.approved.txt | 22 +- .../Baselines/tap.sw.multi.approved.txt | 22 +- .../Baselines/teamcity.sw.approved.txt | 11 + .../Baselines/teamcity.sw.multi.approved.txt | 11 + tests/SelfTest/Baselines/xml.sw.approved.txt | 2161 +++++++++-------- .../Baselines/xml.sw.multi.approved.txt | 2161 +++++++++-------- 21 files changed, 2559 insertions(+), 2261 deletions(-) diff --git a/src/catch2/reporters/catch_reporter_automake.cpp b/src/catch2/reporters/catch_reporter_automake.cpp index 0660d092fc..993b594b85 100644 --- a/src/catch2/reporters/catch_reporter_automake.cpp +++ b/src/catch2/reporters/catch_reporter_automake.cpp @@ -17,7 +17,9 @@ namespace Catch { void AutomakeReporter::testCaseEnded(TestCaseStats const& _testCaseStats) { // Possible values to emit are PASS, XFAIL, SKIP, FAIL, XPASS and ERROR. m_stream << ":test-result: "; - if (_testCaseStats.totals.assertions.allPassed()) { + if ( _testCaseStats.totals.testCases.skipped > 0 ) { + m_stream << "SKIP"; + } else if (_testCaseStats.totals.assertions.allPassed()) { m_stream << "PASS"; } else if (_testCaseStats.totals.assertions.allOk()) { m_stream << "XFAIL"; diff --git a/src/catch2/reporters/catch_reporter_compact.cpp b/src/catch2/reporters/catch_reporter_compact.cpp index 53c3ea53b2..643626eac1 100644 --- a/src/catch2/reporters/catch_reporter_compact.cpp +++ b/src/catch2/reporters/catch_reporter_compact.cpp @@ -105,11 +105,15 @@ class AssertionPrinter { printIssue("explicitly"); printRemainingMessages(Colour::None); break; + case ResultWas::ExplicitSkip: + printResultType(Colour::Skip, "skipped"_sr); + printMessage(); + printRemainingMessages(); + break; // These cases are here to prevent compiler warnings case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: - case ResultWas::ExplicitSkip: printResultType(Colour::Error, "** internal error **"); break; } @@ -221,7 +225,7 @@ class AssertionPrinter { // Drop out if result was successful and we're not printing those if( !m_config->includeSuccessfulResults() && result.isOk() ) { - if( result.getResultType() != ResultWas::Warning ) + if( result.getResultType() != ResultWas::Warning && result.getResultType() != ResultWas::ExplicitSkip ) return; printInfoMessages = false; } diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp index 0f94a3ef11..22d6526fa9 100644 --- a/src/catch2/reporters/catch_reporter_junit.cpp +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -132,6 +132,7 @@ namespace Catch { xml.writeAttribute( "name"_sr, stats.runInfo.name ); xml.writeAttribute( "errors"_sr, unexpectedExceptions ); xml.writeAttribute( "failures"_sr, stats.totals.assertions.failed-unexpectedExceptions ); + xml.writeAttribute( "skipped"_sr, stats.totals.assertions.skipped ); xml.writeAttribute( "tests"_sr, stats.totals.assertions.total() ); xml.writeAttribute( "hostname"_sr, "tbd"_sr ); // !TBD if( m_config->showDurations() == ShowDurations::Never ) @@ -244,7 +245,8 @@ namespace Catch { void JunitReporter::writeAssertion( AssertionStats const& stats ) { AssertionResult const& result = stats.assertionResult; - if( !result.isOk() ) { + if ( !result.isOk() || + result.getResultType() == ResultWas::ExplicitSkip ) { std::string elementName; switch( result.getResultType() ) { case ResultWas::ThrewException: @@ -256,7 +258,9 @@ namespace Catch { case ResultWas::DidntThrowException: elementName = "failure"; break; - + case ResultWas::ExplicitSkip: + elementName = "skipped"; + break; // We should never see these here: case ResultWas::Info: case ResultWas::Warning: @@ -264,7 +268,6 @@ namespace Catch { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: - case ResultWas::ExplicitSkip: elementName = "internalError"; break; } @@ -275,7 +278,9 @@ namespace Catch { xml.writeAttribute( "type"_sr, result.getTestMacroName() ); ReusableStringStream rss; - if (stats.totals.assertions.total() > 0) { + if ( result.getResultType() == ResultWas::ExplicitSkip ) { + rss << "SKIPPED\n"; + } else { rss << "FAILED" << ":\n"; if (result.hasExpression()) { rss << " "; @@ -286,8 +291,6 @@ namespace Catch { rss << "with expansion:\n"; rss << TextFlow::Column(result.getExpandedExpression()).indent(2) << '\n'; } - } else { - rss << '\n'; } if( !result.getMessage().empty() ) diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp index 5ba3a32391..ac59c87fdd 100644 --- a/src/catch2/reporters/catch_reporter_sonarqube.cpp +++ b/src/catch2/reporters/catch_reporter_sonarqube.cpp @@ -97,7 +97,8 @@ namespace Catch { void SonarQubeReporter::writeAssertion(AssertionStats const& stats, bool okToFail) { AssertionResult const& result = stats.assertionResult; - if (!result.isOk()) { + if ( !result.isOk() || + result.getResultType() == ResultWas::ExplicitSkip ) { std::string elementName; if (okToFail) { elementName = "skipped"; @@ -108,15 +109,13 @@ namespace Catch { elementName = "error"; break; case ResultWas::ExplicitFailure: - elementName = "failure"; - break; case ResultWas::ExpressionFailed: - elementName = "failure"; - break; case ResultWas::DidntThrowException: elementName = "failure"; break; - + case ResultWas::ExplicitSkip: + elementName = "skipped"; + break; // We should never see these here: case ResultWas::Info: case ResultWas::Warning: @@ -124,7 +123,6 @@ namespace Catch { case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: - case ResultWas::ExplicitSkip: elementName = "internalError"; break; } @@ -137,7 +135,9 @@ namespace Catch { xml.writeAttribute("message"_sr, messageRss.str()); ReusableStringStream textRss; - if (stats.totals.assertions.total() > 0) { + if ( result.getResultType() == ResultWas::ExplicitSkip ) { + textRss << "SKIPPED\n"; + } else { textRss << "FAILED:\n"; if (result.hasExpression()) { textRss << '\t' << result.getExpressionInMacro() << '\n'; diff --git a/src/catch2/reporters/catch_reporter_tap.cpp b/src/catch2/reporters/catch_reporter_tap.cpp index 4f031524e3..d125711104 100644 --- a/src/catch2/reporters/catch_reporter_tap.cpp +++ b/src/catch2/reporters/catch_reporter_tap.cpp @@ -100,11 +100,16 @@ namespace Catch { printIssue("explicitly"_sr); printRemainingMessages(Colour::None); break; + case ResultWas::ExplicitSkip: + printResultType(tapPassedString); + printIssue(" # SKIP"_sr); + printMessage(); + printRemainingMessages(); + break; // These cases are here to prevent compiler warnings case ResultWas::Unknown: case ResultWas::FailureBit: case ResultWas::Exception: - case ResultWas::ExplicitSkip: printResultType("** internal error **"_sr); break; } diff --git a/src/catch2/reporters/catch_reporter_teamcity.cpp b/src/catch2/reporters/catch_reporter_teamcity.cpp index 4e53871db3..320728007e 100644 --- a/src/catch2/reporters/catch_reporter_teamcity.cpp +++ b/src/catch2/reporters/catch_reporter_teamcity.cpp @@ -59,7 +59,8 @@ namespace Catch { void TeamCityReporter::assertionEnded(AssertionStats const& assertionStats) { AssertionResult const& result = assertionStats.assertionResult; - if (!result.isOk()) { + if ( !result.isOk() || + result.getResultType() == ResultWas::ExplicitSkip ) { ReusableStringStream msg; if (!m_headerPrintedForThisSection) @@ -84,12 +85,14 @@ namespace Catch { case ResultWas::ExplicitFailure: msg << "explicit failure"; break; + case ResultWas::ExplicitSkip: + msg << "explicit skip"; + break; // We shouldn't get here because of the isOk() test case ResultWas::Ok: case ResultWas::Info: case ResultWas::Warning: - case ResultWas::ExplicitSkip: CATCH_ERROR("Internal error in TeamCity reporter"); // These cases are here to prevent compiler warnings case ResultWas::Unknown: @@ -112,18 +115,16 @@ namespace Catch { " " << result.getExpandedExpression() << '\n'; } - if (currentTestCaseInfo->okToFail()) { + if ( result.getResultType() == ResultWas::ExplicitSkip ) { + m_stream << "##teamcity[testIgnored"; + } else if ( currentTestCaseInfo->okToFail() ) { msg << "- failure ignore as test marked as 'ok to fail'\n"; - m_stream << "##teamcity[testIgnored" - << " name='" << escape(currentTestCaseInfo->name) << '\'' - << " message='" << escape(msg.str()) << '\'' - << "]\n"; + m_stream << "##teamcity[testIgnored"; } else { - m_stream << "##teamcity[testFailed" - << " name='" << escape(currentTestCaseInfo->name) << '\'' - << " message='" << escape(msg.str()) << '\'' - << "]\n"; + m_stream << "##teamcity[testFailed"; } + m_stream << " name='" << escape( currentTestCaseInfo->name ) << '\'' + << " message='" << escape( msg.str() ) << '\'' << "]\n"; } m_stream.flush(); } diff --git a/src/catch2/reporters/catch_reporter_xml.cpp b/src/catch2/reporters/catch_reporter_xml.cpp index 57fa1cabe2..03ffff2c22 100644 --- a/src/catch2/reporters/catch_reporter_xml.cpp +++ b/src/catch2/reporters/catch_reporter_xml.cpp @@ -108,9 +108,10 @@ namespace Catch { } // Drop out if result was successful but we're not printing them. - if( !includeResults && result.getResultType() != ResultWas::Warning ) + if ( !includeResults && result.getResultType() != ResultWas::Warning && + result.getResultType() != ResultWas::ExplicitSkip ) { return; - + } // Print the expression if there is one. if( result.hasExpression() ) { @@ -153,6 +154,12 @@ namespace Catch { m_xml.writeText( result.getMessage() ); m_xml.endElement(); break; + case ResultWas::ExplicitSkip: + m_xml.startElement( "Skip" ); + writeSourceInfo( result.getSourceInfo() ); + m_xml.writeText( result.getMessage() ); + m_xml.endElement(); + break; default: break; } @@ -168,6 +175,7 @@ namespace Catch { e.writeAttribute( "successes"_sr, sectionStats.assertions.passed ); e.writeAttribute( "failures"_sr, sectionStats.assertions.failed ); e.writeAttribute( "expectedFailures"_sr, sectionStats.assertions.failedButOk ); + e.writeAttribute( "skipped"_sr, sectionStats.assertions.skipped > 0 ); if ( m_config->showDurations() == ShowDurations::Always ) e.writeAttribute( "durationInSeconds"_sr, sectionStats.durationInSeconds ); @@ -180,6 +188,7 @@ namespace Catch { StreamingReporterBase::testCaseEnded( testCaseStats ); XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResult" ); e.writeAttribute( "success"_sr, testCaseStats.totals.assertions.allOk() ); + e.writeAttribute( "skips"_sr, testCaseStats.totals.assertions.skipped ); if ( m_config->showDurations() == ShowDurations::Always ) e.writeAttribute( "durationInSeconds"_sr, m_testCaseTimer.getElapsedSeconds() ); @@ -197,11 +206,13 @@ namespace Catch { m_xml.scopedElement( "OverallResults" ) .writeAttribute( "successes"_sr, testRunStats.totals.assertions.passed ) .writeAttribute( "failures"_sr, testRunStats.totals.assertions.failed ) - .writeAttribute( "expectedFailures"_sr, testRunStats.totals.assertions.failedButOk ); + .writeAttribute( "expectedFailures"_sr, testRunStats.totals.assertions.failedButOk ) + .writeAttribute( "skips"_sr, testRunStats.totals.assertions.skipped ); m_xml.scopedElement( "OverallResultsCases") .writeAttribute( "successes"_sr, testRunStats.totals.testCases.passed ) .writeAttribute( "failures"_sr, testRunStats.totals.testCases.failed ) - .writeAttribute( "expectedFailures"_sr, testRunStats.totals.testCases.failedButOk ); + .writeAttribute( "expectedFailures"_sr, testRunStats.totals.testCases.failedButOk ) + .writeAttribute( "skips"_sr, testRunStats.totals.testCases.skipped ); m_xml.endElement(); } diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 595b5a5273..64e92396a1 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -297,7 +297,7 @@ Message from section two :test-result: PASS X/level/1/b :test-result: PASS XmlEncode :test-result: PASS XmlWriter writes boolean attributes as true/false -:test-result: XFAIL a succeeding test can still be skipped +:test-result: SKIP a succeeding test can still be skipped :test-result: PASS analyse no analysis :test-result: PASS array -> toString :test-result: PASS benchmark function call @@ -310,7 +310,7 @@ Message from section two :test-result: PASS comparisons between const int variables :test-result: PASS comparisons between int variables :test-result: PASS convertToBits -:test-result: XFAIL dynamic skipping works with generators +:test-result: SKIP dynamic skipping works with generators :test-result: PASS empty tags are not allowed :test-result: PASS erfc_inv :test-result: PASS estimate_clock_resolution @@ -361,11 +361,11 @@ b1! :test-result: PASS run_for_at_least, chronometer :test-result: PASS run_for_at_least, int :test-result: FAIL second tag -:test-result: XFAIL sections can be skipped dynamically at runtime +:test-result: SKIP sections can be skipped dynamically at runtime :test-result: FAIL send a single char to INFO :test-result: FAIL sends information to INFO :test-result: PASS shortened hide tags are split apart -:test-result: XFAIL skipped tests can optionally provide a reason +:test-result: SKIP skipped tests can optionally provide a reason :test-result: PASS splitString :test-result: FAIL stacks unscoped info in loops :test-result: PASS startsWith @@ -387,7 +387,7 @@ b1! :test-result: PASS strlen3 :test-result: PASS tables :test-result: PASS tags with dots in later positions are not parsed as hidden -:test-result: XFAIL tests can be skipped dynamically at runtime +:test-result: SKIP tests can be skipped dynamically at runtime :test-result: FAIL thrown std::strings are translated :test-result: PASS toString on const wchar_t const pointer returns the string contents :test-result: PASS toString on const wchar_t pointer returns the string contents diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index 023d4b4e50..d6f5ebe92b 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -290,7 +290,7 @@ :test-result: PASS X/level/1/b :test-result: PASS XmlEncode :test-result: PASS XmlWriter writes boolean attributes as true/false -:test-result: XFAIL a succeeding test can still be skipped +:test-result: SKIP a succeeding test can still be skipped :test-result: PASS analyse no analysis :test-result: PASS array -> toString :test-result: PASS benchmark function call @@ -303,7 +303,7 @@ :test-result: PASS comparisons between const int variables :test-result: PASS comparisons between int variables :test-result: PASS convertToBits -:test-result: XFAIL dynamic skipping works with generators +:test-result: SKIP dynamic skipping works with generators :test-result: PASS empty tags are not allowed :test-result: PASS erfc_inv :test-result: PASS estimate_clock_resolution @@ -350,11 +350,11 @@ :test-result: PASS run_for_at_least, chronometer :test-result: PASS run_for_at_least, int :test-result: FAIL second tag -:test-result: XFAIL sections can be skipped dynamically at runtime +:test-result: SKIP sections can be skipped dynamically at runtime :test-result: FAIL send a single char to INFO :test-result: FAIL sends information to INFO :test-result: PASS shortened hide tags are split apart -:test-result: XFAIL skipped tests can optionally provide a reason +:test-result: SKIP skipped tests can optionally provide a reason :test-result: PASS splitString :test-result: FAIL stacks unscoped info in loops :test-result: PASS startsWith @@ -376,7 +376,7 @@ :test-result: PASS strlen3 :test-result: PASS tables :test-result: PASS tags with dots in later positions are not parsed as hidden -:test-result: XFAIL tests can be skipped dynamically at runtime +:test-result: SKIP tests can be skipped dynamically at runtime :test-result: FAIL thrown std::strings are translated :test-result: PASS toString on const wchar_t const pointer returns the string contents :test-result: PASS toString on const wchar_t pointer returns the string contents diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index bb78e29b5f..1d8ea9a81d 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -2061,7 +2061,7 @@ Xml.tests.cpp:: passed: stream.str(), ContainsSubstring(R"(attr1="t " ( contains: "attr1="true"" and contains: "attr2="false"" ) Skip.tests.cpp:: passed: -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: InternalBenchmark.tests.cpp:: passed: analysis.mean.point.count() == 23 for: 23.0 == 23 InternalBenchmark.tests.cpp:: passed: analysis.mean.lower_bound.count() == 23 for: 23.0 == 23 InternalBenchmark.tests.cpp:: passed: analysis.mean.upper_bound.count() == 23 for: 23.0 == 23 @@ -2151,9 +2151,9 @@ FloatingPoint.tests.cpp:: passed: convertToBits( -0. ) == ( 1ULL << 9223372036854775808 (0x) FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: 'skipping because answer = 41' Skip.tests.cpp:: passed: -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: 'skipping because answer = 43' Tag.tests.cpp:: passed: Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.103560) == Approx(-0.09203687623843015) for: -0.0920368762 == Approx( -0.0920368762 ) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.067400) == Approx(-0.05980291115763361) for: -0.0598029112 == Approx( -0.0598029112 ) @@ -2164,12 +2164,12 @@ Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Skip.tests.cpp:: failed: 3 == 4 -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: Skip.tests.cpp:: failed: explicitly -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: Skip.tests.cpp:: failed: explicitly -Skip.tests.cpp:: ** internal error **: -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: +Skip.tests.cpp:: skipped: Skip.tests.cpp:: failed: explicitly loose text artifact Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -2230,7 +2230,7 @@ Misc.tests.cpp:: passed: b != a for: 2 != 1 Misc.tests.cpp:: passed: a != b for: 1 != 2 a! b1! -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: ! Tricky.tests.cpp:: passed: s == "7" for: "7" == "7" Tricky.tests.cpp:: passed: ti == typeid(int) for: {?} == {?} @@ -2317,12 +2317,12 @@ InternalBenchmark.tests.cpp:: passed: Timing.elapsed >= time for: 1 InternalBenchmark.tests.cpp:: passed: Timing.result == Timing.iterations + 17 for: 145 == 145 InternalBenchmark.tests.cpp:: passed: Timing.iterations >= time.count() for: 128 >= 100 Skip.tests.cpp:: passed: -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: Skip.tests.cpp:: passed: Misc.tests.cpp:: failed: false with 1 message: '3' Message.tests.cpp:: failed: false with 2 messages: 'hi' and 'i := 7' Tag.tests.cpp:: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: 'skipping because answer = 43' StringManip.tests.cpp:: passed: splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } StringManip.tests.cpp:: passed: splitStringRef("abc", ','), Equals(std::vector{"abc"}) for: { abc } Equals: { abc } StringManip.tests.cpp:: passed: splitStringRef("abc,def", ','), Equals(std::vector{"abc", "def"}) for: { abc, def } Equals: { abc, def } @@ -2388,7 +2388,7 @@ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_cast(std::get<1>(data)) for: 6 == 6 Tag.tests.cpp:: passed: testcase.tags.size() == 1 for: 1 == 1 Tag.tests.cpp:: passed: testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: Exception.tests.cpp:: failed: unexpected exception with message: 'Why would you throw a std::string?' Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 8d0c48291f..1b45055324 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -2054,7 +2054,7 @@ Xml.tests.cpp:: passed: stream.str(), ContainsSubstring(R"(attr1="t " ( contains: "attr1="true"" and contains: "attr2="false"" ) Skip.tests.cpp:: passed: -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: InternalBenchmark.tests.cpp:: passed: analysis.mean.point.count() == 23 for: 23.0 == 23 InternalBenchmark.tests.cpp:: passed: analysis.mean.lower_bound.count() == 23 for: 23.0 == 23 InternalBenchmark.tests.cpp:: passed: analysis.mean.upper_bound.count() == 23 for: 23.0 == 23 @@ -2144,9 +2144,9 @@ FloatingPoint.tests.cpp:: passed: convertToBits( -0. ) == ( 1ULL << 9223372036854775808 (0x) FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 FloatingPoint.tests.cpp:: passed: convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: 'skipping because answer = 41' Skip.tests.cpp:: passed: -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: 'skipping because answer = 43' Tag.tests.cpp:: passed: Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.103560) == Approx(-0.09203687623843015) for: -0.0920368762 == Approx( -0.0920368762 ) InternalBenchmark.tests.cpp:: passed: erfc_inv(1.067400) == Approx(-0.05980291115763361) for: -0.0598029112 == Approx( -0.0598029112 ) @@ -2157,12 +2157,12 @@ Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Skip.tests.cpp:: failed: 3 == 4 -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: Skip.tests.cpp:: failed: explicitly -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: Skip.tests.cpp:: failed: explicitly -Skip.tests.cpp:: ** internal error **: -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: +Skip.tests.cpp:: skipped: Skip.tests.cpp:: failed: explicitly Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' Clara.tests.cpp:: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function::value' @@ -2220,7 +2220,7 @@ Misc.tests.cpp:: passed: a < b for: 1 < 2 Misc.tests.cpp:: passed: a != b for: 1 != 2 Misc.tests.cpp:: passed: b != a for: 2 != 1 Misc.tests.cpp:: passed: a != b for: 1 != 2 -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: Tricky.tests.cpp:: passed: s == "7" for: "7" == "7" Tricky.tests.cpp:: passed: ti == typeid(int) for: {?} == {?} InternalBenchmark.tests.cpp:: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 ) @@ -2306,12 +2306,12 @@ InternalBenchmark.tests.cpp:: passed: Timing.elapsed >= time for: 1 InternalBenchmark.tests.cpp:: passed: Timing.result == Timing.iterations + 17 for: 145 == 145 InternalBenchmark.tests.cpp:: passed: Timing.iterations >= time.count() for: 128 >= 100 Skip.tests.cpp:: passed: -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: Skip.tests.cpp:: passed: Misc.tests.cpp:: failed: false with 1 message: '3' Message.tests.cpp:: failed: false with 2 messages: 'hi' and 'i := 7' Tag.tests.cpp:: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: 'skipping because answer = 43' StringManip.tests.cpp:: passed: splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } StringManip.tests.cpp:: passed: splitStringRef("abc", ','), Equals(std::vector{"abc"}) for: { abc } Equals: { abc } StringManip.tests.cpp:: passed: splitStringRef("abc,def", ','), Equals(std::vector{"abc", "def"}) for: { abc, def } Equals: { abc, def } @@ -2377,7 +2377,7 @@ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_ Generators.tests.cpp:: passed: strlen(std::get<0>(data)) == static_cast(std::get<1>(data)) for: 6 == 6 Tag.tests.cpp:: passed: testcase.tags.size() == 1 for: 1 == 1 Tag.tests.cpp:: passed: testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag -Skip.tests.cpp:: ** internal error **: +Skip.tests.cpp:: skipped: Exception.tests.cpp:: failed: unexpected exception with message: 'Why would you throw a std::string?' Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" Misc.tests.cpp:: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load"" diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index 28cd5c399d..5908fe2c4d 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1591,7 +1591,12 @@ at Exception.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + @@ -1626,7 +1631,18 @@ at Misc.tests.cpp: - + + +SKIPPED +skipping because answer = 41 +at Skip.tests.cpp: + + +SKIPPED +skipping because answer = 43 +at Skip.tests.cpp: + + @@ -1640,18 +1656,35 @@ FAILED: CHECK( 3 == 4 ) at Skip.tests.cpp: + +SKIPPED +at Skip.tests.cpp: + FAILED: at Skip.tests.cpp: + +SKIPPED +at Skip.tests.cpp: + FAILED: at Skip.tests.cpp: + +SKIPPED +at Skip.tests.cpp: + + + + +SKIPPED +at Skip.tests.cpp: + - @@ -1771,7 +1804,12 @@ at Misc.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + a! @@ -1831,7 +1869,12 @@ at Message.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + @@ -1851,7 +1894,13 @@ at Message.tests.cpp: - + + +SKIPPED +skipping because answer = 43 +at Skip.tests.cpp: + + @@ -1896,7 +1945,12 @@ at Message.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + FAILED: diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index d143f532d0..5a82d08791 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -1590,7 +1590,12 @@ at Exception.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + @@ -1625,7 +1630,18 @@ at Misc.tests.cpp: - + + +SKIPPED +skipping because answer = 41 +at Skip.tests.cpp: + + +SKIPPED +skipping because answer = 43 +at Skip.tests.cpp: + + @@ -1639,18 +1655,35 @@ FAILED: CHECK( 3 == 4 ) at Skip.tests.cpp: + +SKIPPED +at Skip.tests.cpp: + FAILED: at Skip.tests.cpp: + +SKIPPED +at Skip.tests.cpp: + FAILED: at Skip.tests.cpp: + +SKIPPED +at Skip.tests.cpp: + + + + +SKIPPED +at Skip.tests.cpp: + - @@ -1770,7 +1803,12 @@ at Misc.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + a! @@ -1830,7 +1868,12 @@ at Message.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + @@ -1850,7 +1893,13 @@ at Message.tests.cpp: - + + +SKIPPED +skipping because answer = 43 +at Skip.tests.cpp: + + @@ -1895,7 +1944,12 @@ at Message.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + FAILED: diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index d83952d82d..fcf21a1c7b 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -1833,39 +1833,93 @@ at Misc.tests.cpp: - - + + +SKIPPED +at Skip.tests.cpp: + + + + +SKIPPED +skipping because answer = 41 +at Skip.tests.cpp: + + +SKIPPED +skipping because answer = 43 +at Skip.tests.cpp: + + FAILED: CHECK( 3 == 4 ) +at Skip.tests.cpp: + + +SKIPPED at Skip.tests.cpp: FAILED: +at Skip.tests.cpp: + + +SKIPPED at Skip.tests.cpp: FAILED: +at Skip.tests.cpp: + + +SKIPPED +at Skip.tests.cpp: + + + + +SKIPPED at Skip.tests.cpp: - FAILED: at Skip.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + - + + +SKIPPED +at Skip.tests.cpp: + + - - + + +SKIPPED +skipping because answer = 43 +at Skip.tests.cpp: + + + + +SKIPPED +at Skip.tests.cpp: + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 7ac035fb15..fdf890e281 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -1832,39 +1832,93 @@ at Misc.tests.cpp: - - + + +SKIPPED +at Skip.tests.cpp: + + + + +SKIPPED +skipping because answer = 41 +at Skip.tests.cpp: + + +SKIPPED +skipping because answer = 43 +at Skip.tests.cpp: + + FAILED: CHECK( 3 == 4 ) +at Skip.tests.cpp: + + +SKIPPED at Skip.tests.cpp: FAILED: +at Skip.tests.cpp: + + +SKIPPED at Skip.tests.cpp: FAILED: +at Skip.tests.cpp: + + +SKIPPED +at Skip.tests.cpp: + + + + +SKIPPED at Skip.tests.cpp: - FAILED: at Skip.tests.cpp: - + + +SKIPPED +at Skip.tests.cpp: + + - + + +SKIPPED +at Skip.tests.cpp: + + - - + + +SKIPPED +skipping because answer = 43 +at Skip.tests.cpp: + + + + +SKIPPED +at Skip.tests.cpp: + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index cca4616030..a8b3b693b5 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -3612,7 +3612,7 @@ ok {test-number} - stream.str(), ContainsSubstring(R"(attr1="true")") && Contain # a succeeding test can still be skipped ok {test-number} - # a succeeding test can still be skipped -** internal error ** 1802 - +ok {test-number} - # SKIP # analyse no analysis ok {test-number} - analysis.mean.point.count() == 23 for: 23.0 == 23 # analyse no analysis @@ -3784,11 +3784,11 @@ ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == # convertToBits ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 # dynamic skipping works with generators -** internal error ** 1888 - +ok {test-number} - # SKIP 'skipping because answer = 41' # dynamic skipping works with generators ok {test-number} - # dynamic skipping works with generators -** internal error ** 1890 - +ok {test-number} - # SKIP 'skipping because answer = 43' # empty tags are not allowed ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) # erfc_inv @@ -3810,17 +3810,17 @@ ok {test-number} - # failed assertions before SKIP cause test case to fail not ok {test-number} - 3 == 4 # failed assertions before SKIP cause test case to fail -** internal error ** 1901 - +ok {test-number} - # SKIP # failing for some generator values causes entire test case to fail not ok {test-number} - explicitly # failing for some generator values causes entire test case to fail -** internal error ** 1903 - +ok {test-number} - # SKIP # failing for some generator values causes entire test case to fail not ok {test-number} - explicitly # failing for some generator values causes entire test case to fail -** internal error ** 1905 - +ok {test-number} - # SKIP # failing in some unskipped sections causes entire test case to fail -** internal error ** 1906 - +ok {test-number} - # SKIP # failing in some unskipped sections causes entire test case to fail not ok {test-number} - explicitly loose text artifact @@ -3935,7 +3935,7 @@ ok {test-number} - a != b for: 1 != 2 a! b1! # nested sections can be skipped dynamically at runtime -** internal error ** 1962 - +ok {test-number} - # SKIP ! # non streamable - with conv. op ok {test-number} - s == "7" for: "7" == "7" @@ -4104,7 +4104,7 @@ ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100 # sections can be skipped dynamically at runtime ok {test-number} - # sections can be skipped dynamically at runtime -** internal error ** 2046 - +ok {test-number} - # SKIP # sections can be skipped dynamically at runtime ok {test-number} - # send a single char to INFO @@ -4114,7 +4114,7 @@ not ok {test-number} - false with 2 messages: 'hi' and 'i := 7' # shortened hide tags are split apart ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) # skipped tests can optionally provide a reason -** internal error ** 2051 - +ok {test-number} - # SKIP 'skipping because answer = 43' # splitString ok {test-number} - splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } # splitString @@ -4198,7 +4198,7 @@ ok {test-number} - testcase.tags.size() == 1 for: 1 == 1 # tags with dots in later positions are not parsed as hidden ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag # tests can be skipped dynamically at runtime -** internal error ** 2093 - +ok {test-number} - # SKIP # thrown std::strings are translated not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?' # toString on const wchar_t const pointer returns the string contents diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 4217c05ff7..ad69ec762c 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -3605,7 +3605,7 @@ ok {test-number} - stream.str(), ContainsSubstring(R"(attr1="true")") && Contain # a succeeding test can still be skipped ok {test-number} - # a succeeding test can still be skipped -** internal error ** 1802 - +ok {test-number} - # SKIP # analyse no analysis ok {test-number} - analysis.mean.point.count() == 23 for: 23.0 == 23 # analyse no analysis @@ -3777,11 +3777,11 @@ ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == # convertToBits ok {test-number} - convertToBits( std::numeric_limits::denorm_min() ) == 1 for: 1 == 1 # dynamic skipping works with generators -** internal error ** 1888 - +ok {test-number} - # SKIP 'skipping because answer = 41' # dynamic skipping works with generators ok {test-number} - # dynamic skipping works with generators -** internal error ** 1890 - +ok {test-number} - # SKIP 'skipping because answer = 43' # empty tags are not allowed ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) # erfc_inv @@ -3803,17 +3803,17 @@ ok {test-number} - # failed assertions before SKIP cause test case to fail not ok {test-number} - 3 == 4 # failed assertions before SKIP cause test case to fail -** internal error ** 1901 - +ok {test-number} - # SKIP # failing for some generator values causes entire test case to fail not ok {test-number} - explicitly # failing for some generator values causes entire test case to fail -** internal error ** 1903 - +ok {test-number} - # SKIP # failing for some generator values causes entire test case to fail not ok {test-number} - explicitly # failing for some generator values causes entire test case to fail -** internal error ** 1905 - +ok {test-number} - # SKIP # failing in some unskipped sections causes entire test case to fail -** internal error ** 1906 - +ok {test-number} - # SKIP # failing in some unskipped sections causes entire test case to fail not ok {test-number} - explicitly # is_unary_function @@ -3925,7 +3925,7 @@ ok {test-number} - b != a for: 2 != 1 # nested SECTION tests ok {test-number} - a != b for: 1 != 2 # nested sections can be skipped dynamically at runtime -** internal error ** 1962 - +ok {test-number} - # SKIP # non streamable - with conv. op ok {test-number} - s == "7" for: "7" == "7" # non-copyable objects @@ -4093,7 +4093,7 @@ ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100 # sections can be skipped dynamically at runtime ok {test-number} - # sections can be skipped dynamically at runtime -** internal error ** 2046 - +ok {test-number} - # SKIP # sections can be skipped dynamically at runtime ok {test-number} - # send a single char to INFO @@ -4103,7 +4103,7 @@ not ok {test-number} - false with 2 messages: 'hi' and 'i := 7' # shortened hide tags are split apart ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} ) # skipped tests can optionally provide a reason -** internal error ** 2051 - +ok {test-number} - # SKIP 'skipping because answer = 43' # splitString ok {test-number} - splitStringRef("", ','), Equals(std::vector()) for: { } Equals: { } # splitString @@ -4187,7 +4187,7 @@ ok {test-number} - testcase.tags.size() == 1 for: 1 == 1 # tags with dots in later positions are not parsed as hidden ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag # tests can be skipped dynamically at runtime -** internal error ** 2093 - +ok {test-number} - # SKIP # thrown std::strings are translated not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?' # toString on const wchar_t const pointer returns the string contents diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index 4c97262046..df406058d9 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -723,6 +723,7 @@ ##teamcity[testStarted name='XmlWriter writes boolean attributes as true/false'] ##teamcity[testFinished name='XmlWriter writes boolean attributes as true/false' duration="{duration}"] ##teamcity[testStarted name='a succeeding test can still be skipped'] +##teamcity[testIgnored name='a succeeding test can still be skipped' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='a succeeding test can still be skipped' duration="{duration}"] ##teamcity[testStarted name='analyse no analysis'] ##teamcity[testFinished name='analyse no analysis' duration="{duration}"] @@ -751,6 +752,8 @@ ##teamcity[testStarted name='convertToBits'] ##teamcity[testFinished name='convertToBits' duration="{duration}"] ##teamcity[testStarted name='dynamic skipping works with generators'] +##teamcity[testIgnored name='dynamic skipping works with generators' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "skipping because answer = 41"'] +##teamcity[testIgnored name='dynamic skipping works with generators' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "skipping because answer = 43"'] ##teamcity[testFinished name='dynamic skipping works with generators' duration="{duration}"] ##teamcity[testStarted name='empty tags are not allowed'] ##teamcity[testFinished name='empty tags are not allowed' duration="{duration}"] @@ -762,12 +765,16 @@ ##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"] ##teamcity[testStarted name='failed assertions before SKIP cause test case to fail'] ##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='failed assertions before SKIP cause test case to fail' duration="{duration}"] ##teamcity[testStarted name='failing for some generator values causes entire test case to fail'] ##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='failing for some generator values causes entire test case to fail' duration="{duration}"] ##teamcity[testStarted name='failing in some unskipped sections causes entire test case to fail'] +##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nskipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nnot skipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] ##teamcity[testFinished name='failing in some unskipped sections causes entire test case to fail' duration="{duration}"] ##teamcity[testStarted name='first tag'] @@ -817,6 +824,7 @@ loose text artifact ##teamcity[testStarted name='nested SECTION tests'] ##teamcity[testFinished name='nested SECTION tests' duration="{duration}"] ##teamcity[testStarted name='nested sections can be skipped dynamically at runtime'] +##teamcity[testIgnored name='nested sections can be skipped dynamically at runtime' message='-------------------------------------------------------------------------------|nB|nB2|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testStdOut name='nested sections can be skipped dynamically at runtime' out='a!|nb1!|n!|n'] ##teamcity[testFinished name='nested sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='non streamable - with conv. op'] @@ -865,6 +873,7 @@ loose text artifact ##teamcity[testStarted name='second tag'] ##teamcity[testFinished name='second tag' duration="{duration}"] ##teamcity[testStarted name='sections can be skipped dynamically at runtime'] +##teamcity[testIgnored name='sections can be skipped dynamically at runtime' message='-------------------------------------------------------------------------------|nskipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='send a single char to INFO'] ##teamcity[testFailed name='send a single char to INFO' message='Misc.tests.cpp:|n...............................................................................|n|nMisc.tests.cpp:|nexpression failed with message:|n "3"|n REQUIRE( false )|nwith expansion:|n false|n'] @@ -875,6 +884,7 @@ loose text artifact ##teamcity[testStarted name='shortened hide tags are split apart'] ##teamcity[testFinished name='shortened hide tags are split apart' duration="{duration}"] ##teamcity[testStarted name='skipped tests can optionally provide a reason'] +##teamcity[testIgnored name='skipped tests can optionally provide a reason' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "skipping because answer = 43"'] ##teamcity[testFinished name='skipped tests can optionally provide a reason' duration="{duration}"] ##teamcity[testStarted name='splitString'] ##teamcity[testFinished name='splitString' duration="{duration}"] @@ -921,6 +931,7 @@ loose text artifact ##teamcity[testStarted name='tags with dots in later positions are not parsed as hidden'] ##teamcity[testFinished name='tags with dots in later positions are not parsed as hidden' duration="{duration}"] ##teamcity[testStarted name='tests can be skipped dynamically at runtime'] +##teamcity[testIgnored name='tests can be skipped dynamically at runtime' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='tests can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='thrown std::strings are translated'] ##teamcity[testFailed name='thrown std::strings are translated' message='Exception.tests.cpp:|n...............................................................................|n|nException.tests.cpp:|nunexpected exception with message:|n "Why would you throw a std::string?"'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index 9a6b61ed3a..1aea9f9b88 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -723,6 +723,7 @@ ##teamcity[testStarted name='XmlWriter writes boolean attributes as true/false'] ##teamcity[testFinished name='XmlWriter writes boolean attributes as true/false' duration="{duration}"] ##teamcity[testStarted name='a succeeding test can still be skipped'] +##teamcity[testIgnored name='a succeeding test can still be skipped' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='a succeeding test can still be skipped' duration="{duration}"] ##teamcity[testStarted name='analyse no analysis'] ##teamcity[testFinished name='analyse no analysis' duration="{duration}"] @@ -751,6 +752,8 @@ ##teamcity[testStarted name='convertToBits'] ##teamcity[testFinished name='convertToBits' duration="{duration}"] ##teamcity[testStarted name='dynamic skipping works with generators'] +##teamcity[testIgnored name='dynamic skipping works with generators' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "skipping because answer = 41"'] +##teamcity[testIgnored name='dynamic skipping works with generators' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "skipping because answer = 43"'] ##teamcity[testFinished name='dynamic skipping works with generators' duration="{duration}"] ##teamcity[testStarted name='empty tags are not allowed'] ##teamcity[testFinished name='empty tags are not allowed' duration="{duration}"] @@ -762,12 +765,16 @@ ##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"] ##teamcity[testStarted name='failed assertions before SKIP cause test case to fail'] ##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='failed assertions before SKIP cause test case to fail' duration="{duration}"] ##teamcity[testStarted name='failing for some generator values causes entire test case to fail'] ##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='failing for some generator values causes entire test case to fail' duration="{duration}"] ##teamcity[testStarted name='failing in some unskipped sections causes entire test case to fail'] +##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nskipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nnot skipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] ##teamcity[testFinished name='failing in some unskipped sections causes entire test case to fail' duration="{duration}"] ##teamcity[testStarted name='first tag'] @@ -816,6 +823,7 @@ ##teamcity[testStarted name='nested SECTION tests'] ##teamcity[testFinished name='nested SECTION tests' duration="{duration}"] ##teamcity[testStarted name='nested sections can be skipped dynamically at runtime'] +##teamcity[testIgnored name='nested sections can be skipped dynamically at runtime' message='-------------------------------------------------------------------------------|nB|nB2|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testStdOut name='nested sections can be skipped dynamically at runtime' out='a!|nb1!|n!|n'] ##teamcity[testFinished name='nested sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='non streamable - with conv. op'] @@ -864,6 +872,7 @@ ##teamcity[testStarted name='second tag'] ##teamcity[testFinished name='second tag' duration="{duration}"] ##teamcity[testStarted name='sections can be skipped dynamically at runtime'] +##teamcity[testIgnored name='sections can be skipped dynamically at runtime' message='-------------------------------------------------------------------------------|nskipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='sections can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='send a single char to INFO'] ##teamcity[testFailed name='send a single char to INFO' message='Misc.tests.cpp:|n...............................................................................|n|nMisc.tests.cpp:|nexpression failed with message:|n "3"|n REQUIRE( false )|nwith expansion:|n false|n'] @@ -874,6 +883,7 @@ ##teamcity[testStarted name='shortened hide tags are split apart'] ##teamcity[testFinished name='shortened hide tags are split apart' duration="{duration}"] ##teamcity[testStarted name='skipped tests can optionally provide a reason'] +##teamcity[testIgnored name='skipped tests can optionally provide a reason' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "skipping because answer = 43"'] ##teamcity[testFinished name='skipped tests can optionally provide a reason' duration="{duration}"] ##teamcity[testStarted name='splitString'] ##teamcity[testFinished name='splitString' duration="{duration}"] @@ -920,6 +930,7 @@ ##teamcity[testStarted name='tags with dots in later positions are not parsed as hidden'] ##teamcity[testFinished name='tags with dots in later positions are not parsed as hidden' duration="{duration}"] ##teamcity[testStarted name='tests can be skipped dynamically at runtime'] +##teamcity[testIgnored name='tests can be skipped dynamically at runtime' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip'] ##teamcity[testFinished name='tests can be skipped dynamically at runtime' duration="{duration}"] ##teamcity[testStarted name='thrown std::strings are translated'] ##teamcity[testFailed name='thrown std::strings are translated' message='Exception.tests.cpp:|n...............................................................................|n|nException.tests.cpp:|nunexpected exception with message:|n "Why would you throw a std::string?"'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 0d4ea7a819..8db8915131 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -20,7 +20,7 @@ 0 == 0 - + @@ -71,10 +71,10 @@ {?} >= {?} - + - + @@ -105,16 +105,16 @@ 0 == 0 - + - +
- +
- + @@ -125,7 +125,7 @@ [1403 helper] == [1403 helper] - + @@ -136,13 +136,13 @@ This info message starts with a linebreak This warning message starts with a linebreak - + 1514 - + This would not be caught previously @@ -160,7 +160,7 @@ Nor would this true - + @@ -187,7 +187,7 @@ Nor would this !false - +
@@ -215,7 +215,7 @@ Nor would this !false - +
@@ -226,9 +226,9 @@ Nor would this true - +
- + @@ -247,7 +247,7 @@ Nor would this 6 < 7 - + @@ -282,11 +282,11 @@ Nor would this 2 != 4 - +
- +
@@ -297,7 +297,7 @@ Nor would this 1 - +
@@ -308,7 +308,7 @@ Nor would this 2 - +
@@ -319,9 +319,9 @@ Nor would this 3 - +
- +
@@ -333,7 +333,7 @@ Nor would this 1 - +
@@ -351,7 +351,7 @@ Nor would this 3 - + @@ -378,11 +378,11 @@ Nor would this 3 - +
- +
i := 1 @@ -394,7 +394,7 @@ Nor would this k := 5
- +
i := 1 @@ -406,7 +406,7 @@ Nor would this k := 6
- +
i := 1 @@ -427,7 +427,7 @@ Nor would this k := 6
- +
i := 2 @@ -439,7 +439,7 @@ Nor would this k := 5
- +
i := 2 @@ -451,7 +451,7 @@ Nor would this k := 6
- +
i := 2 @@ -471,7 +471,7 @@ Nor would this k := 6 - + @@ -618,16 +618,16 @@ Nor would this 3 - + - + - + - + @@ -646,7 +646,7 @@ Nor would this 0.0 not is within 1 ULPs of -4.9406564584124654e-324 ([-9.8813129168249309e-324, -0.0000000000000000e+00]) - + @@ -665,7 +665,7 @@ Nor would this 0.0f not is within 1 ULPs of -1.40129846e-45f ([-2.80259693e-45, -0.00000000e+00]) - +
@@ -675,7 +675,7 @@ Nor would this expected exception - +
@@ -692,7 +692,7 @@ Nor would this expected exception - +
@@ -706,9 +706,9 @@ Nor would this thisThrows() - +
- + @@ -719,7 +719,7 @@ Nor would this 42 == {?} - + @@ -778,7 +778,7 @@ Nor would this true - + @@ -797,7 +797,7 @@ Nor would this 1 == 1 - + @@ -811,25 +811,25 @@ Nor would this {?} == 4 - +
- +
- +
- +
- +
- +
- + @@ -896,7 +896,7 @@ Nor would this !(1 == 1) - + @@ -963,7 +963,7 @@ Nor would this !(1 == 2) - +
@@ -983,7 +983,7 @@ Nor would this true == true - +
@@ -1002,7 +1002,7 @@ Nor would this false == false - +
@@ -1013,7 +1013,7 @@ Nor would this true - +
@@ -1024,7 +1024,7 @@ Nor would this true - +
@@ -1043,9 +1043,9 @@ Nor would this !false - +
- + @@ -1696,7 +1696,7 @@ Nor would this 3 < 9 - + @@ -1707,7 +1707,7 @@ Nor would this "hello" == "world" - + @@ -1718,7 +1718,7 @@ Nor would this "hello" == "hello" - + @@ -1729,7 +1729,7 @@ Nor would this 0 == 1 - + @@ -1740,7 +1740,7 @@ Nor would this 0 == 1 - + @@ -1751,7 +1751,7 @@ Nor would this 0 == 1 - + @@ -1762,7 +1762,7 @@ Nor would this 0 == 1 - + @@ -1773,7 +1773,7 @@ Nor would this 0 == 0 - + @@ -1784,7 +1784,7 @@ Nor would this 0 == 0 - + @@ -1795,7 +1795,7 @@ Nor would this 0 == 0 - + @@ -1806,7 +1806,7 @@ Nor would this 0 == 0 - + @@ -1817,7 +1817,7 @@ Nor would this 6 < 2 - + @@ -1828,7 +1828,7 @@ Nor would this 2 < 2 - + @@ -1839,7 +1839,7 @@ Nor would this 6 < 2 - + @@ -1850,7 +1850,7 @@ Nor would this 2 < 2 - + @@ -1861,7 +1861,7 @@ Nor would this 6 >= 2 - + @@ -1872,7 +1872,7 @@ Nor would this 2 >= 2 - + @@ -1883,7 +1883,7 @@ Nor would this 6 >= 2 - + @@ -1894,7 +1894,7 @@ Nor would this 2 >= 2 - + @@ -1905,7 +1905,7 @@ Nor would this 1.0 == 2 - + @@ -1916,7 +1916,7 @@ Nor would this 1.0f == 2 - + @@ -1927,7 +1927,7 @@ Nor would this 1 == 2 - + @@ -1938,7 +1938,7 @@ Nor would this 1.0 == 1 - + @@ -1949,7 +1949,7 @@ Nor would this 1.0f == 1 - + @@ -1960,7 +1960,7 @@ Nor would this 1 == 1 - + @@ -1971,7 +1971,7 @@ Nor would this 1 == 0 - + @@ -1982,7 +1982,7 @@ Nor would this 3 == 0 - + @@ -1993,7 +1993,7 @@ Nor would this 6 == 0 - + @@ -2004,7 +2004,7 @@ Nor would this 1 > 0 - + @@ -2015,7 +2015,7 @@ Nor would this 3 > 0 - + @@ -2026,7 +2026,7 @@ Nor would this 6 > 0 - + @@ -2037,7 +2037,7 @@ Nor would this 1 == 2 - + @@ -2048,7 +2048,7 @@ Nor would this 1 == 1 - + @@ -2059,7 +2059,7 @@ Nor would this 0 == 0 - + @@ -2070,7 +2070,7 @@ Nor would this 0 == 0 - + @@ -2081,7 +2081,7 @@ Nor would this 0 == 0 - + @@ -2092,7 +2092,7 @@ Nor would this 0 == 0 - + @@ -2103,7 +2103,7 @@ Nor would this 42 > 0 - + @@ -2114,7 +2114,7 @@ Nor would this 9 > 0 - + @@ -2125,7 +2125,7 @@ Nor would this 42 > 0 - + @@ -2136,7 +2136,7 @@ Nor would this 9 > 0 - + @@ -2187,19 +2187,19 @@ Nor would this 1.23 == Approx( 1.0 ) - +
- +
- +
to infinity and beyond - + @@ -2218,7 +2218,7 @@ Nor would this {?} == {?} - + @@ -2269,10 +2269,10 @@ Nor would this 100.3 == Approx( 100.0 ) - + - + @@ -2291,7 +2291,7 @@ Nor would this 8 == 8 - + @@ -2313,10 +2313,10 @@ Nor would this unexpected exception - + - + @@ -2375,7 +2375,7 @@ Nor would this Approx(0).epsilon(1.0001), std::domain_error - + @@ -2418,7 +2418,7 @@ Nor would this 245.5f == Approx( 245.25 ) - + @@ -2437,7 +2437,7 @@ Nor would this 3.1428571429 != Approx( 3.141 ) - + @@ -2456,7 +2456,7 @@ Nor would this 1.23 == Approx( 1.231 ) - + @@ -2475,7 +2475,7 @@ Nor would this 0.0f == Approx( 0.0 ) - + @@ -2494,7 +2494,7 @@ Nor would this 0 == Approx( 0.0 ) - + @@ -2537,7 +2537,7 @@ Nor would this 1.234 == Approx( 1.2339999676 ) - +
@@ -2557,7 +2557,7 @@ Nor would this 1 not matches predicate: "always false" - +
@@ -2576,9 +2576,9 @@ Nor would this "This wouldn't pass" not matches undescribed predicate - +
- + @@ -2621,7 +2621,7 @@ Nor would this !(Val: 1 ^ Val: 1) - + @@ -2650,9 +2650,9 @@ Nor would this true - + - + @@ -2680,11 +2680,11 @@ Nor would this true - + - + - +
@@ -2712,7 +2712,7 @@ Nor would this { 4, 5, 6 } not contains element 1 - +
@@ -2739,7 +2739,7 @@ Nor would this { 4, 5, 6 } not contains element 0 - +
@@ -2750,7 +2750,7 @@ Nor would this { "abc", "abcd", "abcde" } contains element 4 - +
@@ -2769,7 +2769,7 @@ Nor would this { 1, 2, 3, 4, 5 } not contains element 8 - +
@@ -2788,7 +2788,7 @@ Nor would this { 1, 2, 3 } not contains element 9 - +
@@ -2799,9 +2799,9 @@ Nor would this { 1.0, 2.0, 3.0, 0.0 } contains element matching is within 0.5 of 0.5 - +
- +
@@ -2853,7 +2853,7 @@ Nor would this { } is empty - +
@@ -2864,7 +2864,7 @@ Nor would this {?} not is empty - +
@@ -2875,9 +2875,9 @@ Nor would this {?} is empty - +
- + @@ -2901,7 +2901,7 @@ Nor would this a == 1 := true - + @@ -2925,7 +2925,7 @@ Nor would this (2, 3) := 3 - + @@ -2961,7 +2961,7 @@ Nor would this '{' := '{' - +
@@ -2976,7 +2976,7 @@ Nor would this true - +
@@ -2990,9 +2990,9 @@ Nor would this true - +
- +
@@ -3012,7 +3012,7 @@ Nor would this !false - +
@@ -3063,9 +3063,9 @@ Nor would this !false - +
- +
@@ -3093,7 +3093,7 @@ Nor would this !false - +
@@ -3128,9 +3128,9 @@ Nor would this true - +
- +
@@ -3166,7 +3166,7 @@ Nor would this '\f' == '\f' - +
@@ -3209,7 +3209,7 @@ Nor would this 'Z' == 'Z' - +
@@ -3252,9 +3252,9 @@ Nor would this 5 == 5 - +
- + @@ -3273,7 +3273,7 @@ Nor would this "foo" == "foo" - +
@@ -3285,7 +3285,7 @@ Nor would this !{?} - +
@@ -3304,9 +3304,9 @@ Nor would this { "aaa", "bbb" } == { "aaa", "bbb" } - +
- +
@@ -3318,7 +3318,7 @@ Nor would this true - +
@@ -3341,7 +3341,7 @@ Using code: 0 " - +
@@ -3364,9 +3364,9 @@ C " - +
- + @@ -3393,7 +3393,7 @@ C 1 ( equals: (int) 1 or (string) "1" and equals: (long long) 1 and equals: (T) 1 and equals: true ) - + @@ -3420,7 +3420,7 @@ C 1 ( equals: (int) 1 or (string) "1" or equals: (long long) 1 or equals: (T) 1 or equals: true ) - + @@ -3455,10 +3455,10 @@ C 1 equals: (int) 1 or (string) "1" - + - + @@ -3485,7 +3485,7 @@ C 1 ( equals: (int) 1 or (string) "1" or not equals: (long long) 1 ) - + @@ -3544,7 +3544,7 @@ C "foobar" ( ( starts with: "foo" and ends with: "bar" ) or Equals: { 'o', 'o', 'f', 'b', 'a', 'r' } ) - + @@ -3555,7 +3555,7 @@ C { 1, 2, 3 } ( Equals: { 1, 2, 3 } or Equals: { 0, 1, 2 } or Equals: { 4, 5, 6 } ) - + @@ -3654,7 +3654,7 @@ C { 1, 2 } == { 1, 2 } - + @@ -3673,7 +3673,7 @@ C 0x == 0x - + @@ -3708,7 +3708,7 @@ C !({?} != {?}) - + @@ -3807,7 +3807,7 @@ C Approx( 11.0 ) >= StrongDoubleTypedef(10) - + @@ -3818,7 +3818,7 @@ C 54 == 54 - + @@ -3869,7 +3869,7 @@ C -2147483648 > 2 - + @@ -3976,7 +3976,7 @@ C 4294967295 (0x) > 4 - +
@@ -4004,7 +4004,7 @@ C true - +
@@ -4031,9 +4031,9 @@ C true - +
- +
@@ -4061,7 +4061,7 @@ C true - +
@@ -4088,9 +4088,9 @@ C true - +
- + @@ -4109,7 +4109,7 @@ C "this string contains 'abc' as a substring" contains: "STRING" - +
@@ -4121,7 +4121,7 @@ C 1 == 1 - +
@@ -4132,7 +4132,7 @@ C 1 == 1 - +
@@ -4143,7 +4143,7 @@ C 1 == 1 - +
@@ -4154,7 +4154,7 @@ C 1 == 1 - +
@@ -4165,7 +4165,7 @@ C 1 == 1 - +
@@ -4176,7 +4176,7 @@ C 1 == 1 - +
@@ -4187,7 +4187,7 @@ C 1 == 1 - +
@@ -4198,7 +4198,7 @@ C 1 == 1 - +
@@ -4209,7 +4209,7 @@ C 1 == 1 - +
@@ -4220,7 +4220,7 @@ C 1 == 1 - +
@@ -4231,7 +4231,7 @@ C 1 == 1 - +
@@ -4242,7 +4242,7 @@ C 1 == 1 - +
@@ -4261,9 +4261,9 @@ C 6 == 6 - +
- + @@ -4274,7 +4274,7 @@ C true - + @@ -4288,7 +4288,7 @@ C custom exception - not std - + @@ -4302,13 +4302,13 @@ C custom exception - not std - + custom std exception - + @@ -4327,7 +4327,7 @@ C 0.00001 != Approx( 0.0000001 ) - + @@ -4356,7 +4356,7 @@ C "{** unexpected enum value **}" - + @@ -4367,7 +4367,7 @@ C true - + @@ -4378,7 +4378,7 @@ C Catch::TestCaseInfo( "", { "fake test name", "[]" }, dummySourceLineInfo ) - + @@ -4397,7 +4397,7 @@ C "this string contains 'abc' as a substring" ends with: "this" (case insensitive) - + @@ -4442,7 +4442,7 @@ C "Value2" == "Value2" - + @@ -4461,7 +4461,7 @@ C "Blue" == "Blue" - + @@ -4472,7 +4472,7 @@ C 101.01 != Approx( 100.0 ) - + @@ -4579,7 +4579,7 @@ C 1.3 == Approx( 1.301 ) - + @@ -4638,7 +4638,7 @@ C 1.3 == Approx( 1.3 ) - + @@ -4657,7 +4657,7 @@ C "this string contains 'abc' as a substring" equals: "this string contains 'abc' as a substring" (case insensitive) - + @@ -4676,7 +4676,7 @@ C "this string contains 'abc' as a substring" equals: "something else" (case insensitive) - + @@ -4707,7 +4707,7 @@ C "StringMakerException" - +
@@ -4727,7 +4727,7 @@ C doesNotThrow(), SpecialException, ExceptionMatcher{ 1 } - +
@@ -4752,7 +4752,7 @@ C Unknown exception - +
@@ -4771,9 +4771,9 @@ C SpecialException::what special exception has value of 1 - +
- + @@ -4792,7 +4792,7 @@ C SpecialException::what special exception has value of 2 - + @@ -4827,7 +4827,7 @@ C SpecialException::what matches "starts with: "Special"" - +
@@ -4839,7 +4839,7 @@ C "expected exception" equals: "expected exception" - +
@@ -4850,7 +4850,7 @@ C "expected exception" equals: "expected exception" (case insensitive) - +
@@ -4885,9 +4885,9 @@ C "expected exception" contains: "except" (case insensitive) - +
- + @@ -4922,7 +4922,7 @@ C SpecialException::what exception message matches "SpecialException::what" - + @@ -4955,17 +4955,17 @@ C expected exception - + This is a failure - + - + @@ -4974,7 +4974,7 @@ C This message appears in the output - + @@ -5017,7 +5017,7 @@ C 3628800 (0x) == 3628800 (0x) - +
@@ -5062,9 +5062,9 @@ C 0.0 and 2.22507e-308 are within 2.22045e-12% of each other - +
- +
@@ -5131,7 +5131,7 @@ C -10.0 is within 0.5 of -9.6 - +
@@ -5190,7 +5190,7 @@ C -0.0 is within 0 ULPs of 0.0000000000000000e+00 ([0.0000000000000000e+00, 0.0000000000000000e+00]) - +
@@ -5217,7 +5217,7 @@ C 0.0001 ( is within 0.001 of 0.0 or and 0 are within 10% of each other ) - +
@@ -5268,9 +5268,9 @@ C WithinRel( 1., 1. ), std::domain_error - +
- +
@@ -5315,9 +5315,9 @@ C 0.0f and 1.17549e-38 are within 0.00119209% of each other - +
- +
@@ -5392,7 +5392,7 @@ C -10.0f is within 0.5 of -9.6000003815 - +
@@ -5459,7 +5459,7 @@ C -0.0f is within 0 ULPs of 0.00000000e+00f ([0.00000000e+00, 0.00000000e+00]) - +
@@ -5486,7 +5486,7 @@ C 0.0001f ( is within 0.001 of 0.0 or and 0 are within 10% of each other ) - +
@@ -5545,9 +5545,9 @@ C WithinRel( 1.f, 1.f ), std::domain_error - +
- +
@@ -5560,9 +5560,9 @@ C 0 == 0 - +
- +
@@ -5574,9 +5574,9 @@ C 0 == 0 - +
- +
@@ -5588,9 +5588,9 @@ C 0 == 0 - +
- +
@@ -5602,9 +5602,9 @@ C filter([] (int) {return false; }, value(1)), Catch::GeneratorException - +
- +
@@ -5615,7 +5615,7 @@ C 1 < 4 - +
@@ -5626,7 +5626,7 @@ C 2 < 4 - +
@@ -5637,7 +5637,7 @@ C 3 < 4 - +
@@ -5649,9 +5649,9 @@ C 0 == 0 - +
- +
@@ -5663,9 +5663,9 @@ C 0 == 0 - +
- +
@@ -5677,9 +5677,9 @@ C 0 == 0 - +
- +
@@ -5691,9 +5691,9 @@ C 1 == 1 - +
- +
@@ -5705,9 +5705,9 @@ C 1 == 1 - +
- +
@@ -5719,9 +5719,9 @@ C 1 == 1 - +
- +
@@ -5733,9 +5733,9 @@ C 1 == 1 - +
- +
@@ -5747,9 +5747,9 @@ C 1 == 1 - +
- +
@@ -5761,9 +5761,9 @@ C 1 == 1 - +
- +
@@ -5774,7 +5774,7 @@ C 1 > 0 - +
@@ -5785,7 +5785,7 @@ C 2 > 0 - +
@@ -5796,7 +5796,7 @@ C 3 > 0 - +
@@ -5807,7 +5807,7 @@ C 1 > 0 - +
@@ -5818,7 +5818,7 @@ C 2 > 0 - +
@@ -5829,7 +5829,7 @@ C 3 > 0 - +
@@ -5849,9 +5849,9 @@ C 1 == 1 - +
- +
@@ -5871,9 +5871,9 @@ C 2 == 2 - +
- +
@@ -5893,9 +5893,9 @@ C 3 == 3 - +
- +
@@ -5923,9 +5923,9 @@ C 1 < 3 - +
- +
@@ -5953,9 +5953,9 @@ C 2 < 3 - +
- +
@@ -5967,9 +5967,9 @@ C 0 == 0 - +
- +
@@ -5981,9 +5981,9 @@ C 0 == 0 - +
- +
@@ -5995,9 +5995,9 @@ C 0 == 0 - +
- +
@@ -6009,11 +6009,11 @@ C chunk(2, value(1)), Catch::GeneratorException - +
- +
- +
@@ -6025,7 +6025,7 @@ C -3 < 1 - +
@@ -6036,7 +6036,7 @@ C -2 < 1 - +
@@ -6047,7 +6047,7 @@ C -1 < 1 - +
@@ -6058,7 +6058,7 @@ C 4 > 1 - +
@@ -6069,7 +6069,7 @@ C 4 > 2 - +
@@ -6080,7 +6080,7 @@ C 4 > 3 - +
@@ -6091,7 +6091,7 @@ C -3 < 2 - +
@@ -6102,7 +6102,7 @@ C -2 < 2 - +
@@ -6113,7 +6113,7 @@ C -1 < 2 - +
@@ -6124,7 +6124,7 @@ C 8 > 1 - +
@@ -6135,7 +6135,7 @@ C 8 > 2 - +
@@ -6146,7 +6146,7 @@ C 8 > 3 - +
@@ -6157,7 +6157,7 @@ C -3 < 3 - +
@@ -6168,7 +6168,7 @@ C -2 < 3 - +
@@ -6179,7 +6179,7 @@ C -1 < 3 - +
@@ -6190,7 +6190,7 @@ C 12 > 1 - +
@@ -6201,7 +6201,7 @@ C 12 > 2 - +
@@ -6212,9 +6212,9 @@ C 12 > 3 - +
- +
@@ -6234,7 +6234,7 @@ C !false - +
@@ -6285,7 +6285,7 @@ C !false - +
@@ -6368,7 +6368,7 @@ C !false - +
@@ -6427,7 +6427,7 @@ C !false - +
@@ -6463,9 +6463,9 @@ C !false - +
- +
@@ -6501,9 +6501,9 @@ C !false - +
- +
@@ -6523,9 +6523,9 @@ C filter([](int) { return false; }, values({ 1, 2, 3 })), Catch::GeneratorException - +
- +
@@ -6561,9 +6561,9 @@ C !false - +
- +
@@ -6583,9 +6583,9 @@ C !false - +
- +
@@ -6636,7 +6636,7 @@ C !false - +
@@ -6687,7 +6687,7 @@ C !false - +
@@ -6707,9 +6707,9 @@ C !false - +
- +
@@ -6809,9 +6809,9 @@ C !false - +
- +
@@ -6880,11 +6880,11 @@ C !false - +
- +
- +
@@ -6953,11 +6953,11 @@ C !false - +
- +
- +
@@ -7027,13 +7027,13 @@ C !false - +
- +
- + - +
@@ -7103,13 +7103,13 @@ C !false - +
- +
- + - +
@@ -7195,13 +7195,13 @@ C !false - +
- +
- + - +
@@ -7663,13 +7663,13 @@ C !false - +
- +
- + - +
@@ -7815,13 +7815,13 @@ C !false - +
- +
- + - +
@@ -7967,13 +7967,13 @@ C !false - +
- +
- + - +
@@ -8043,13 +8043,13 @@ C !false - +
- +
- + - +
@@ -8119,13 +8119,13 @@ C !false - +
- +
- + - +
@@ -8211,15 +8211,15 @@ C !false - +
- +
- + - + - + @@ -8254,7 +8254,7 @@ C 1.23 >= Approx( 1.24 ) - + @@ -8267,7 +8267,7 @@ C 130711275 (0x) - + @@ -8280,7 +8280,7 @@ C 3422778688 (0x) - +
@@ -8294,7 +8294,7 @@ C 2668622104 (0x) - +
@@ -8307,7 +8307,7 @@ C 3916075712 (0x) - +
@@ -8320,9 +8320,9 @@ C 3429949824 (0x) - +
- + @@ -8335,7 +8335,7 @@ C 3422778688 (0x) - + @@ -8344,7 +8344,7 @@ C this is a warning - + @@ -8361,7 +8361,7 @@ C 2 == 1 - + @@ -8426,7 +8426,7 @@ C 2 == 2 - + @@ -8583,7 +8583,7 @@ C 10 < 10 - + @@ -8626,7 +8626,7 @@ C 5 != 5 - + @@ -8717,7 +8717,7 @@ C 5 != 6 - + @@ -8728,7 +8728,7 @@ C true - + @@ -8763,10 +8763,10 @@ C 1.23 <= Approx( 1.22 ) - + - + @@ -8777,7 +8777,7 @@ C "this string contains 'abc' as a substring" ( contains: "string" and contains: "abc" and contains: "substring" and contains: "contains" ) - + @@ -8796,7 +8796,7 @@ C "some completely different text that contains one common word" ( contains: "string" or contains: "different" or contains: "random" ) - + @@ -8807,7 +8807,7 @@ C "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "substring" ) - + @@ -8818,7 +8818,7 @@ C "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "random" ) - + @@ -8829,7 +8829,7 @@ C "this string contains 'abc' as a substring" not contains: "different" - + @@ -8840,44 +8840,44 @@ C "this string contains 'abc' as a substring" not contains: "substring" - +
- +
- +
- +
- +
- +
- +
- +
- +
- +
- +
- + @@ -8896,7 +8896,7 @@ C "expected exception" equals: "should fail" - + @@ -8909,7 +8909,7 @@ C { "Hello", "world", "Goodbye", "world" } - + @@ -8977,7 +8977,7 @@ C true == true - + @@ -9044,9 +9044,9 @@ C true == true - + - + @@ -9177,19 +9177,19 @@ C 99 > -6 - + This one ran - + custom exception - + @@ -9216,10 +9216,10 @@ C !{?} - + - + @@ -9374,7 +9374,7 @@ C "hello" <= "a" - + @@ -9513,7 +9513,7 @@ C "hello" > "a" - +
@@ -9567,7 +9567,7 @@ C 1827115164 (0x) - +
@@ -9670,24 +9670,24 @@ C 4261393167 (0x) - +
- +
Message from section one - +
Message from section two - +
- + @@ -9722,7 +9722,7 @@ C ( EvilMatcher() && EvilMatcher() ) || !EvilMatcher() - +
@@ -9758,7 +9758,7 @@ C {?} == {?} - +
@@ -9817,9 +9817,9 @@ C !{?} - +
- + @@ -9846,7 +9846,7 @@ C true - +
@@ -9866,7 +9866,7 @@ C 8 == 8 - +
@@ -9885,7 +9885,7 @@ C "Could not parse '-1' as shard count" contains: "Could not parse '-1' as shard count" - +
@@ -9904,7 +9904,7 @@ C "Shard count must be positive" contains: "Shard count must be positive" - +
@@ -9923,7 +9923,7 @@ C 2 == 2 - +
@@ -9942,7 +9942,7 @@ C "Could not parse '-12' as shard index" contains: "Could not parse '-12' as shard index" - +
@@ -9961,9 +9961,9 @@ C 0 == 0 - +
- + @@ -10032,7 +10032,7 @@ C true - +
@@ -10052,7 +10052,7 @@ C 1 == 1 - +
@@ -10063,7 +10063,7 @@ C !{?} - +
@@ -10082,9 +10082,9 @@ C 3 == 3 - +
- + @@ -10151,7 +10151,7 @@ C 0 != 0x - +
@@ -10171,7 +10171,7 @@ C 13 == 13 - +
@@ -10190,9 +10190,9 @@ C 17 == 17 - +
- + @@ -10203,7 +10203,7 @@ C "foo" matches undescribed predicate - +
@@ -10223,7 +10223,7 @@ C "" == "" - +
@@ -10314,7 +10314,7 @@ C {?} == {?} - +
@@ -10350,9 +10350,9 @@ C true - +
- +
@@ -10388,9 +10388,9 @@ C true - +
- +
@@ -10426,9 +10426,9 @@ C true - +
- +
@@ -10454,9 +10454,9 @@ C { {?} } == { {?} } - +
- +
@@ -10482,9 +10482,9 @@ C { {?} } == { {?} } - +
- +
@@ -10510,9 +10510,9 @@ C { {?} } == { {?} } - +
- +
@@ -10532,9 +10532,9 @@ C "Unrecognized reporter, 'unsupported'. Check available with --list-reporters" contains: "Unrecognized reporter" - +
- +
@@ -10560,9 +10560,9 @@ C { {?} } == { {?} } - +
- +
@@ -10588,9 +10588,9 @@ C { {?} } == { {?} } - +
- +
@@ -10611,11 +10611,11 @@ C { {?}, {?} } == { {?}, {?} } - +
- +
- +
@@ -10636,11 +10636,11 @@ C { {?}, {?} } == { {?}, {?} } - +
- +
- +
@@ -10661,11 +10661,11 @@ C "Only one reporter may have unspecified output file." contains: "Only one reporter may have unspecified output file." - +
- +
- +
@@ -10685,9 +10685,9 @@ C true == true - +
- +
@@ -10707,9 +10707,9 @@ C true - +
- +
@@ -10729,9 +10729,9 @@ C 1 == 1 - +
- +
@@ -10751,9 +10751,9 @@ C 2 == 2 - +
- +
@@ -10773,9 +10773,9 @@ C "Unable to convert 'oops' to destination type" ( contains: "convert" and contains: "oops" ) - +
- +
@@ -10796,11 +10796,11 @@ C 0 == 0 - +
- +
- +
@@ -10821,11 +10821,11 @@ C 1 == 1 - +
- +
- +
@@ -10846,11 +10846,11 @@ C 2 == 2 - +
- +
- +
@@ -10871,11 +10871,11 @@ C 3 == 3 - +
- +
- +
@@ -10896,11 +10896,11 @@ C "keypress argument must be one of: never, start, exit or both. 'sometimes' not recognised" ( contains: "never" and contains: "both" ) - +
- +
- +
@@ -10920,9 +10920,9 @@ C true - +
- +
@@ -10942,9 +10942,9 @@ C true - +
- +
@@ -10964,9 +10964,9 @@ C "filename.ext" == "filename.ext" - +
- +
@@ -10986,9 +10986,9 @@ C "filename.ext" == "filename.ext" - +
- +
@@ -11024,9 +11024,9 @@ C true == true - +
- +
@@ -11046,9 +11046,9 @@ C 0 == 0 - +
- +
@@ -11068,9 +11068,9 @@ C 0 == 0 - +
- +
@@ -11090,9 +11090,9 @@ C 1 == 1 - +
- +
@@ -11112,9 +11112,9 @@ C 3 == 3 - +
- +
@@ -11134,9 +11134,9 @@ C "colour mode must be one of: default, ansi, win32, or none. 'wrong' is not recognised" contains: "colour mode must be one of" - +
- +
@@ -11156,9 +11156,9 @@ C 200 == 200 - +
- +
@@ -11178,9 +11178,9 @@ C 20000 (0x) == 20000 (0x) - +
- +
@@ -11200,9 +11200,9 @@ C 0.99 == Approx( 0.99 ) - +
- +
@@ -11222,9 +11222,9 @@ C true - +
- +
@@ -11244,11 +11244,11 @@ C 10 == 10 - +
- +
- + @@ -11259,7 +11259,7 @@ C 3 >= 1 - + @@ -11270,7 +11270,7 @@ C 2 >= 1 - + @@ -11281,7 +11281,7 @@ C 1 >= 1 - + @@ -11308,7 +11308,7 @@ C Catch::generateRandomSeed(method) - + @@ -11319,7 +11319,7 @@ C Catch::generateRandomSeed(static_cast<Catch::GenerateFrom>(77)) - + @@ -11330,7 +11330,7 @@ C "{ }" == "{ }" - + @@ -11341,7 +11341,7 @@ C Hey, its truthy! - + @@ -11368,7 +11368,7 @@ C "this string contains 'abc' as a substring" matches "this string contains 'abc' as a" case sensitively - + @@ -11379,7 +11379,7 @@ C "'::' is not allowed in reporter name: 'with::doublecolons'" equals: "'::' is not allowed in reporter name: 'with::doublecolons'" - + @@ -11390,7 +11390,7 @@ C { 'a', 'b' } not UnorderedEquals: { 'c', 'b' } - + @@ -11417,7 +11417,7 @@ C " contains: "fakeTag" - + @@ -11442,7 +11442,7 @@ C " contains: "fake reporter" - + @@ -11469,7 +11469,7 @@ C " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11495,7 +11495,7 @@ C " contains: "fakeTag" - + @@ -11520,7 +11520,7 @@ C " contains: "fake reporter" - + @@ -11547,7 +11547,7 @@ C " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11573,7 +11573,7 @@ C " contains: "fakeTag" - + @@ -11598,7 +11598,7 @@ C " contains: "fake reporter" - + @@ -11625,7 +11625,7 @@ C " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11652,7 +11652,7 @@ All available tags: " contains: "fakeTag" - + @@ -11678,7 +11678,7 @@ Available reporters: " contains: "fake reporter" - + @@ -11706,7 +11706,7 @@ All available test cases: " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11733,7 +11733,7 @@ All available tags: " contains: "fakeTag" - + @@ -11759,7 +11759,7 @@ Available reporters: " contains: "fake reporter" - + @@ -11787,7 +11787,7 @@ All available test cases: " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11813,7 +11813,7 @@ All available test cases: " contains: "fakeTag" - + @@ -11838,7 +11838,7 @@ All available test cases: " contains: "fake reporter" - + @@ -11865,7 +11865,7 @@ All available test cases: " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11891,7 +11891,7 @@ All available test cases: " contains: "fakeTag" - + @@ -11916,7 +11916,7 @@ All available test cases: " contains: "fake reporter" - + @@ -11943,7 +11943,7 @@ All available test cases: " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11973,7 +11973,7 @@ All available test cases: </TagsFromMatchingTests>" contains: "fakeTag" - + @@ -12001,7 +12001,7 @@ All available test cases: </AvailableReporters>" contains: "fake reporter" - + @@ -12034,18 +12034,18 @@ All available test cases: </MatchingTests>" ( contains: "fake test name" and contains: "fakeTestTag" ) - + - + - + - + - +
@@ -12067,13 +12067,13 @@ All available test cases: 1 > 0 - +
- + - + - +
@@ -12097,29 +12097,29 @@ All available test cases: true - +
- + - + - + - + - +
- +
- +
- +
- +
@@ -12167,15 +12167,15 @@ All available test cases: 10 >= 10 - +
- + - + - + - +
@@ -12204,16 +12204,16 @@ All available test cases: 0 == 0 - +
- + - + - + - + A string sent directly to stdout @@ -12288,16 +12288,16 @@ A string sent to stderr via clog Approx( 1.23 ) != 1.24 - +
- +
- +
- + Message from section one Message from section two @@ -12321,7 +12321,7 @@ Message from section two "this string contains 'abc' as a substring" starts with: "string" (case insensitive) - +
@@ -12333,7 +12333,7 @@ Message from section two "{ 1 }" == "{ 1 }" - +
@@ -12344,7 +12344,7 @@ Message from section two "{ 3, 2, 1 }" == "{ 3, 2, 1 }" - +
@@ -12357,9 +12357,9 @@ Message from section two "{ { "1:1", "1:2", "1:3" }, { "2:1", "2:2" } }" - +
- + @@ -12426,7 +12426,7 @@ Message from section two "this string contains 'abc' as a substring" ends with: " substring" (case insensitive) - +
@@ -12454,7 +12454,7 @@ Message from section two 0 == 0 - +
@@ -12489,7 +12489,7 @@ Message from section two "hello" == "hello" - +
@@ -12508,7 +12508,7 @@ Message from section two original.data() - +
@@ -12519,7 +12519,7 @@ Message from section two "original string" == "original string" - +
@@ -12530,7 +12530,7 @@ Message from section two "original string" == "original string" - +
@@ -12566,9 +12566,9 @@ Message from section two hello == "hello" - +
- +
@@ -12588,9 +12588,9 @@ Message from section two 0 == 0 - +
- +
@@ -12602,9 +12602,9 @@ Message from section two "hello world!" == "hello world!" - +
- +
@@ -12616,9 +12616,9 @@ Message from section two "hello world!" == "hello world!" - +
- +
@@ -12630,9 +12630,9 @@ Message from section two true - +
- +
@@ -12644,9 +12644,9 @@ Message from section two 0 == 0 - +
- +
@@ -12658,9 +12658,9 @@ Message from section two true - +
- +
@@ -12687,7 +12687,7 @@ Message from section two Hello != Hel - +
@@ -12707,9 +12707,9 @@ Message from section two 17 == 17 - +
- +
@@ -12729,9 +12729,9 @@ Message from section two 17 == 17 - +
- +
@@ -12751,9 +12751,9 @@ Message from section two 17 == 17 - +
- +
@@ -12773,9 +12773,9 @@ Message from section two 11 == 11 - +
- +
@@ -12795,9 +12795,9 @@ Message from section two 11 == 11 - +
- +
@@ -12810,7 +12810,7 @@ Message from section two "some string += the stringref contents" - +
@@ -12821,18 +12821,18 @@ Message from section two "abrakadabra" == "abrakadabra" - +
- +
- +
- +
- + @@ -12851,7 +12851,7 @@ Message from section two ""abc"" == ""abc"" - + @@ -12870,7 +12870,7 @@ Message from section two ""abc"" == ""abc"" - + @@ -12889,7 +12889,7 @@ Message from section two ""abc"" == ""abc"" - + @@ -12924,7 +12924,7 @@ Message from section two 1 ns != 1 us - + @@ -12943,7 +12943,7 @@ Message from section two 1 ps != 1 as - + @@ -12956,7 +12956,7 @@ Message from section two {iso8601-timestamp} - + @@ -12974,7 +12974,7 @@ Message from section two " - +
@@ -13018,7 +13018,7 @@ Message from section two Redefined at: file:10" contains: "10" - +
@@ -13053,9 +13053,9 @@ Message from section two registry.add( "[@no square bracket at end", "", Catch::SourceLineInfo( "file", 3 ) ) - +
- + @@ -13074,7 +13074,7 @@ Message from section two { {?}, {?} } ( Contains: {?} and Contains: {?} ) - + @@ -13085,7 +13085,7 @@ Message from section two 1 == 1 - + @@ -13096,7 +13096,7 @@ Message from section two 1 == 1 - + @@ -13107,7 +13107,7 @@ Message from section two 1.0 == 1 - + @@ -13118,7 +13118,7 @@ Message from section two 1 > 0 - + @@ -13129,7 +13129,7 @@ Message from section two 4 > 0 - + @@ -13140,7 +13140,7 @@ Message from section two 1 > 0 - + @@ -13151,7 +13151,7 @@ Message from section two 4 > 0 - + @@ -13162,7 +13162,7 @@ Message from section two 4 > 0 - + @@ -13173,7 +13173,7 @@ Message from section two 1 > 0 - + @@ -13184,7 +13184,7 @@ Message from section two 4 > 0 - + @@ -13220,7 +13220,7 @@ Message from section two 10 >= 10 - + @@ -13264,9 +13264,9 @@ Message from section two 0 == 0 - + - + @@ -13301,7 +13301,7 @@ Message from section two 10 >= 10 - + @@ -13336,9 +13336,9 @@ Message from section two 5 >= 5 - + - + @@ -13374,7 +13374,7 @@ Message from section two 10 >= 10 - + @@ -13418,9 +13418,9 @@ Message from section two 0 == 0 - + - + @@ -13455,7 +13455,7 @@ Message from section two 10 >= 10 - + @@ -13490,9 +13490,9 @@ Message from section two 5 >= 5 - + - + @@ -13528,7 +13528,7 @@ Message from section two 10 >= 10 - + @@ -13572,9 +13572,9 @@ Message from section two 0 == 0 - + - + @@ -13609,7 +13609,7 @@ Message from section two 10 >= 10 - + @@ -13644,9 +13644,9 @@ Message from section two 5 >= 5 - + - + @@ -13682,7 +13682,7 @@ Message from section two 10 >= 10 - + @@ -13726,9 +13726,9 @@ Message from section two 0 == 0 - + - + @@ -13763,7 +13763,7 @@ Message from section two 10 >= 10 - + @@ -13798,9 +13798,9 @@ Message from section two 5 >= 5 - + - + @@ -13836,7 +13836,7 @@ Message from section two 12 >= 12 - + @@ -13880,9 +13880,9 @@ Message from section two 0 == 0 - + - + @@ -13917,7 +13917,7 @@ Message from section two 12 >= 12 - + @@ -13952,9 +13952,9 @@ Message from section two 6 >= 6 - + - + @@ -13990,7 +13990,7 @@ Message from section two 8 >= 8 - + @@ -14034,9 +14034,9 @@ Message from section two 0 == 0 - + - + @@ -14071,7 +14071,7 @@ Message from section two 8 >= 8 - + @@ -14106,9 +14106,9 @@ Message from section two 4 >= 4 - + - + @@ -14144,7 +14144,7 @@ Message from section two 10 >= 10 - + @@ -14188,9 +14188,9 @@ Message from section two 0 == 0 - + - + @@ -14225,7 +14225,7 @@ Message from section two 10 >= 10 - + @@ -14260,9 +14260,9 @@ Message from section two 5 >= 5 - + - + @@ -14298,7 +14298,7 @@ Message from section two 30 >= 30 - + @@ -14342,9 +14342,9 @@ Message from section two 0 == 0 - + - + @@ -14379,7 +14379,7 @@ Message from section two 30 >= 30 - + @@ -14414,9 +14414,9 @@ Message from section two 15 >= 15 - + - + @@ -14435,10 +14435,10 @@ Message from section two {?} == {?} - + - + @@ -14449,10 +14449,10 @@ Message from section two 3221225472 (0x) == 3221225472 - + - + @@ -14487,7 +14487,7 @@ Message from section two false - + @@ -14499,7 +14499,7 @@ Message from section two - + @@ -14511,7 +14511,7 @@ Message from section two - + @@ -14522,7 +14522,7 @@ Message from section two 1 == 2 - +
@@ -14538,7 +14538,7 @@ Message from section two " contains: "[fakeTag]" - +
@@ -14552,7 +14552,7 @@ Message from section two " ( contains: "fake reporter" and contains: "fake description" ) - +
@@ -14568,7 +14568,7 @@ Message from section two " ( contains: "fake test name" and contains: "fakeTestTag" ) - +
@@ -14582,18 +14582,18 @@ Message from section two " ( contains: "fakeListener" and contains: "fake description" ) - +
- + - + For some reason someone is throwing a string literal! - + @@ -14645,7 +14645,7 @@ Message from section two true - + @@ -14745,9 +14745,9 @@ Message from section two true - + - + @@ -14855,9 +14855,9 @@ Message from section two true - + - + @@ -14958,11 +14958,11 @@ Message from section two true - + - + - + @@ -15095,11 +15095,11 @@ Message from section two true - + - + - + @@ -15166,9 +15166,9 @@ Message from section two true - + - + @@ -15251,13 +15251,13 @@ There is no extra whitespace here There is no extra whitespace here - + 3.14 - +
@@ -15269,7 +15269,7 @@ There is no extra whitespace here 3 == 3 - +
@@ -15280,9 +15280,9 @@ There is no extra whitespace here 3 == 3 - +
- +
@@ -15302,7 +15302,7 @@ There is no extra whitespace here { { 0, 1, 2, 3, 5 }, { 4, -3, -2, 5, 0 }, { 0, 0, 0, 5, 0 }, { 0, -5, 0, 5, 0 }, { 1, 0, 0, -1, 5 } } not all match ( contains element 0 and contains element 1 ) - +
@@ -15313,7 +15313,7 @@ There is no extra whitespace here { 1, 2, 3, 4, 5 } all match matches undescribed predicate - +
@@ -15365,9 +15365,9 @@ There is no extra whitespace here true - +
- +
@@ -15419,11 +15419,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -15436,9 +15436,9 @@ There is no extra whitespace here { true, true, true, true, true } contains only true - +
- +
@@ -15450,9 +15450,9 @@ There is no extra whitespace here { } contains only true - +
- +
@@ -15464,9 +15464,9 @@ There is no extra whitespace here { true, true, false, true, true } not contains only true - +
- +
@@ -15478,9 +15478,9 @@ There is no extra whitespace here { false, false, false, false, false } not contains only true - +
- +
@@ -15492,9 +15492,9 @@ There is no extra whitespace here { true, true, true, true, true } contains only true - +
- +
@@ -15506,9 +15506,9 @@ There is no extra whitespace here { true, true, false, true, true } not contains only true - +
- +
@@ -15520,9 +15520,9 @@ There is no extra whitespace here { false, false, false, false, false } not contains only true - +
- +
@@ -15574,9 +15574,9 @@ There is no extra whitespace here true - +
- +
@@ -15628,11 +15628,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -15652,7 +15652,7 @@ There is no extra whitespace here { { 0, 1, 2, 3, 5 }, { 4, -3, -2, 5, 0 }, { 0, 0, 0, 5, 0 }, { 0, -5, 0, 5, 0 }, { 1, 0, 0, -1, 5 } } not any match ( contains element 0 and contains element 10 ) - +
@@ -15663,7 +15663,7 @@ There is no extra whitespace here { 1, 2, 3, 4, 5 } any match matches undescribed predicate - +
@@ -15715,9 +15715,9 @@ There is no extra whitespace here true - +
- +
@@ -15769,11 +15769,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -15786,9 +15786,9 @@ There is no extra whitespace here { true, true, true, true, true } contains at least one true - +
- +
@@ -15800,9 +15800,9 @@ There is no extra whitespace here { } not contains at least one true - +
- +
@@ -15814,9 +15814,9 @@ There is no extra whitespace here { false, false, true, false, false } contains at least one true - +
- +
@@ -15828,9 +15828,9 @@ There is no extra whitespace here { false, false, false, false, false } not contains at least one true - +
- +
@@ -15842,9 +15842,9 @@ There is no extra whitespace here { true, true, true, true, true } contains at least one true - +
- +
@@ -15856,9 +15856,9 @@ There is no extra whitespace here { false, false, true, false, false } contains at least one true - +
- +
@@ -15870,9 +15870,9 @@ There is no extra whitespace here { false, false, false, false, false } not contains at least one true - +
- +
@@ -15924,9 +15924,9 @@ There is no extra whitespace here true - +
- +
@@ -15978,11 +15978,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -16002,7 +16002,7 @@ There is no extra whitespace here { { 0, 1, 2, 3, 5 }, { 4, -3, -2, 5, 0 }, { 0, 0, 0, 5, 0 }, { 0, -5, 0, 5, 0 }, { 1, 0, 0, -1, 5 } } not none match ( contains element 0 and contains element 1 ) - +
@@ -16013,7 +16013,7 @@ There is no extra whitespace here { 1, 2, 3, 4, 5 } none match matches undescribed predicate - +
@@ -16065,9 +16065,9 @@ There is no extra whitespace here true - +
- +
@@ -16119,11 +16119,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -16136,9 +16136,9 @@ There is no extra whitespace here { true, true, true, true, true } not contains no true - +
- +
@@ -16150,9 +16150,9 @@ There is no extra whitespace here { } contains no true - +
- +
@@ -16164,9 +16164,9 @@ There is no extra whitespace here { false, false, true, false, false } not contains no true - +
- +
@@ -16178,9 +16178,9 @@ There is no extra whitespace here { false, false, false, false, false } contains no true - +
- +
@@ -16192,9 +16192,9 @@ There is no extra whitespace here { true, true, true, true, true } not contains no true - +
- +
@@ -16206,9 +16206,9 @@ There is no extra whitespace here { false, false, true, false, false } not contains no true - +
- +
@@ -16220,9 +16220,9 @@ There is no extra whitespace here { false, false, false, false, false } contains no true - +
- +
@@ -16274,9 +16274,9 @@ There is no extra whitespace here true - +
- +
@@ -16328,11 +16328,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -16392,7 +16392,7 @@ There is no extra whitespace here { {?}, {?}, {?} } has size == 3 - +
@@ -16403,7 +16403,7 @@ There is no extra whitespace here {?} has size == 12 - +
@@ -16414,9 +16414,9 @@ There is no extra whitespace here {?} has size == 13 - +
- + @@ -16483,13 +16483,13 @@ There is no extra whitespace here Approx( 1.23 ) != 1.25 - +
- +
- +
@@ -16501,7 +16501,7 @@ There is no extra whitespace here { } is approx: { } - +
@@ -16521,9 +16521,9 @@ There is no extra whitespace here { 1.0, 2.0, 3.0 } is approx: { 1.0, 2.0, 3.0 } - +
- +
@@ -16535,9 +16535,9 @@ There is no extra whitespace here { 1.0, 2.0, 3.0 } not is approx: { 1.0, 2.0, 3.0, 4.0 } - +
- +
@@ -16573,11 +16573,11 @@ There is no extra whitespace here { 1.0, 2.0, 3.0 } is approx: { 1.5, 2.5, 3.5 } - +
- +
- +
@@ -16589,7 +16589,7 @@ There is no extra whitespace here { } is approx: { 1.0, 2.0 } - +
@@ -16600,9 +16600,9 @@ There is no extra whitespace here { 2.0, 4.0, 6.0 } is approx: { 1.0, 3.0, 5.0 } - +
- +
@@ -16630,7 +16630,7 @@ There is no extra whitespace here { 1, 2, 3 } Contains: 2 - +
@@ -16697,7 +16697,7 @@ There is no extra whitespace here { 1, 2, 3 } Contains: { 1, 2 } - +
@@ -16708,7 +16708,7 @@ There is no extra whitespace here { 1, 2, 3 } ( Contains: 1 and Contains: 2 ) - +
@@ -16759,7 +16759,7 @@ There is no extra whitespace here { 1, 2, 3 } Equals: { 1, 2, 3 } - +
@@ -16818,9 +16818,9 @@ There is no extra whitespace here { 1, 3, 2 } UnorderedEquals: { 1, 2, 3 } - +
- +
@@ -16840,7 +16840,7 @@ There is no extra whitespace here { } Contains: 1 - +
@@ -16859,7 +16859,7 @@ There is no extra whitespace here { 1, 2, 3 } Contains: { 1, 2, 4 } - +
@@ -16894,7 +16894,7 @@ There is no extra whitespace here { 1, 2, 3 } Equals: { } - +
@@ -16929,9 +16929,9 @@ There is no extra whitespace here { 3, 1 } UnorderedEquals: { 1, 2, 3 } - +
- + @@ -16958,13 +16958,13 @@ There is no extra whitespace here thisThrows() - + unexpected exception - + @@ -16978,7 +16978,7 @@ There is no extra whitespace here expected exception - + @@ -16992,7 +16992,7 @@ There is no extra whitespace here expected exception - + @@ -17006,31 +17006,31 @@ There is no extra whitespace here expected exception - +
unexpected exception - +
- + - + - + - + - + - +
@@ -17042,7 +17042,7 @@ There is no extra whitespace here "normal string" == "normal string" - +
@@ -17053,7 +17053,7 @@ There is no extra whitespace here "" == "" - +
@@ -17064,7 +17064,7 @@ There is no extra whitespace here "smith &amp; jones" == "smith &amp; jones" - +
@@ -17075,7 +17075,7 @@ There is no extra whitespace here "smith &lt; jones" == "smith &lt; jones" - +
@@ -17096,7 +17096,7 @@ There is no extra whitespace here "smith ]]&gt; jones" - +
@@ -17119,7 +17119,7 @@ There is no extra whitespace here "don't &quot;quote&quot; me on that" - +
@@ -17130,7 +17130,7 @@ There is no extra whitespace here "[\x01]" == "[\x01]" - +
@@ -17141,9 +17141,9 @@ There is no extra whitespace here "[\x7F]" == "[\x7F]" - +
- + @@ -17156,10 +17156,11 @@ There is no extra whitespace here " ( contains: "attr1="true"" and contains: "attr2="false"" ) - + - + + @@ -17266,7 +17267,7 @@ There is no extra whitespace here 0.0 == 0 - + @@ -17293,7 +17294,7 @@ There is no extra whitespace here "{ 42, 250 }" == "{ 42, 250 }" - +
@@ -17337,7 +17338,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17380,9 +17381,9 @@ There is no extra whitespace here 1 == 1 - +
- + @@ -17393,7 +17394,7 @@ There is no extra whitespace here 0x != 0 - + @@ -17412,7 +17413,7 @@ There is no extra whitespace here true - + @@ -17431,7 +17432,7 @@ There is no extra whitespace here false - + @@ -17450,7 +17451,7 @@ There is no extra whitespace here true - + @@ -17469,7 +17470,7 @@ There is no extra whitespace here false - +
@@ -17521,7 +17522,7 @@ There is no extra whitespace here 0 == 0 - +
@@ -17572,7 +17573,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17623,7 +17624,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17674,7 +17675,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17725,7 +17726,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17776,9 +17777,9 @@ There is no extra whitespace here 2 == 2 - +
- + @@ -17813,7 +17814,7 @@ There is no extra whitespace here 1 == 1 - + @@ -17848,7 +17849,7 @@ There is no extra whitespace here 1 == 1 - + @@ -17903,10 +17904,16 @@ There is no extra whitespace here 1 == 1 - + - + + skipping because answer = 41 + + + skipping because answer = 43 + + @@ -17917,7 +17924,7 @@ There is no extra whitespace here Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) - + @@ -17944,7 +17951,7 @@ There is no extra whitespace here 1.3859038243 == Approx( 1.3859038243 ) - + @@ -17963,25 +17970,25 @@ There is no extra whitespace here 0 == 0 - +
- +
- +
- +
- +
- +
- + @@ -17992,50 +17999,54 @@ There is no extra whitespace here 3 == 4 - + + + - + +
- + +
- +
- + - + loose text artifact - + - + Previous info should not be seen - + previous unscoped info SHOULD not be seen - + - + - + @@ -18048,7 +18059,7 @@ loose text artifact 9223372036854775807 (0x) - +
@@ -18060,7 +18071,7 @@ loose text artifact 0 > 1 - +
@@ -18071,7 +18082,7 @@ loose text artifact 1 > 1 - +
@@ -18082,7 +18093,7 @@ loose text artifact 2 > 1 - +
@@ -18093,7 +18104,7 @@ loose text artifact 3 > 1 - +
@@ -18104,7 +18115,7 @@ loose text artifact 4 > 1 - +
@@ -18115,7 +18126,7 @@ loose text artifact 5 > 1 - +
@@ -18126,7 +18137,7 @@ loose text artifact 6 > 1 - +
@@ -18137,7 +18148,7 @@ loose text artifact 7 > 1 - +
@@ -18148,7 +18159,7 @@ loose text artifact 8 > 1 - +
@@ -18159,9 +18170,9 @@ loose text artifact 9 > 1 - +
- + @@ -18252,7 +18263,7 @@ loose text artifact 1 == 0 - + @@ -18263,7 +18274,7 @@ loose text artifact Catch::makeStream( "%debug" ) - +
@@ -18275,7 +18286,7 @@ loose text artifact !false - +
@@ -18286,7 +18297,7 @@ loose text artifact true - +
@@ -18297,9 +18308,9 @@ loose text artifact {?} == {?} - +
- + @@ -18310,7 +18321,7 @@ loose text artifact 19.0 == 19.0 - + @@ -18377,7 +18388,7 @@ loose text artifact 1 == 1 - + @@ -18398,7 +18409,7 @@ loose text artifact they are not cleared after warnings - +
@@ -18411,9 +18422,9 @@ loose text artifact 1 == 2 - +
- +
@@ -18425,9 +18436,9 @@ loose text artifact 1 != 2 - +
- +
@@ -18439,11 +18450,11 @@ loose text artifact 1 < 2 - +
- +
- +
@@ -18472,32 +18483,33 @@ loose text artifact 1 != 2 - +
- + - +
- +
- +
- +
- + +
- +
- +
- + a! b1! @@ -18514,7 +18526,7 @@ b1! "7" == "7" - + @@ -18525,7 +18537,7 @@ b1! {?} == {?} - + @@ -18568,7 +18580,7 @@ b1! 0.088096521 == Approx( 0.088096521 ) - + @@ -18595,10 +18607,10 @@ b1! -1.9599639845 == Approx( -1.9599639845 ) - + - + @@ -18634,7 +18646,7 @@ b1! false - + @@ -18653,7 +18665,7 @@ b1! {null string} == {null string} - + @@ -18664,7 +18676,7 @@ b1! 0 == 0 - + @@ -18677,7 +18689,7 @@ b1! "{ { 42, "Arthur" }, { "Ford", 24 } }" - +
@@ -18689,7 +18701,7 @@ b1! { } Equals: { } - +
@@ -18716,7 +18728,7 @@ b1! { Value1 } Equals: { Value1 } - +
@@ -18743,9 +18755,9 @@ b1! { Value1, Value2, Value3 } Equals: { Value1, Value2, Value3 } - +
- + @@ -18756,7 +18768,7 @@ b1! 0 == 0 - + @@ -18770,7 +18782,7 @@ b1! true - + @@ -18787,7 +18799,7 @@ b1! false - + @@ -18828,7 +18840,7 @@ b1! true - +
@@ -18848,7 +18860,7 @@ b1! 2 != 1 - +
@@ -18859,9 +18871,9 @@ b1! 1 != 2 - +
- +
@@ -18881,7 +18893,7 @@ b1! "azcdefcg" == "azcdefcg" - +
@@ -18900,7 +18912,7 @@ b1! "abzdefzg" == "abzdefzg" - +
@@ -18919,7 +18931,7 @@ b1! "zbcdefcg" == "zbcdefcg" - +
@@ -18938,7 +18950,7 @@ b1! "abcdefcz" == "abcdefcz" - +
@@ -18957,7 +18969,7 @@ b1! "replaced" == "replaced" - +
@@ -18976,7 +18988,7 @@ b1! "abcdefcg" == "abcdefcg" - +
@@ -18995,9 +19007,9 @@ b1! "didn|'t" == "didn|'t" - +
- + @@ -19008,7 +19020,7 @@ b1! Catch::makeStream( "%somestream" ) - + @@ -19091,7 +19103,7 @@ b1! 1000.0 == 1000 (0x) - + @@ -19182,7 +19194,7 @@ b1! 128 >= 100 - + @@ -19273,22 +19285,23 @@ b1! 128 >= 100 - + - +
- +
- + +
- +
- + @@ -19302,7 +19315,7 @@ b1! false - + @@ -19319,7 +19332,7 @@ b1! false - + @@ -19330,10 +19343,13 @@ b1! { {?}, {?} } ( Contains: {?} and Contains: {?} ) - + - + + skipping because answer = 43 + + @@ -19360,7 +19376,7 @@ b1! { abc, def } Equals: { abc, def } - + @@ -19403,7 +19419,7 @@ b1! false - + @@ -19430,7 +19446,7 @@ b1! true - +
@@ -19442,7 +19458,7 @@ b1! "{ }" == "{ }" - +
@@ -19453,7 +19469,7 @@ b1! "{ { "one", 1 } }" == "{ { "one", 1 } }" - +
@@ -19466,9 +19482,9 @@ b1! "{ { "abc", 1 }, { "def", 2 }, { "ghi", 3 } }" - +
- + @@ -19479,7 +19495,7 @@ b1! "{ 34, "xyzzy" }" == "{ 34, "xyzzy" }" - + @@ -19490,7 +19506,7 @@ b1! "{ 34, "xyzzy" }" == "{ 34, "xyzzy" }" - +
@@ -19502,7 +19518,7 @@ b1! "{ }" == "{ }" - +
@@ -19513,7 +19529,7 @@ b1! "{ "one" }" == "{ "one" }" - +
@@ -19526,9 +19542,9 @@ b1! "{ "abc", "def", "ghi" }" - +
- + @@ -19541,7 +19557,7 @@ b1! "{ { "green", 55 } }" - + @@ -19560,7 +19576,7 @@ b1! true - + @@ -19599,7 +19615,7 @@ b1! "{?}" == "{?}" - + @@ -19612,7 +19628,7 @@ b1! "StringMaker<has_maker>" - + @@ -19625,7 +19641,7 @@ b1! "StringMaker<has_maker_and_operator>" - + @@ -19636,7 +19652,7 @@ b1! "{?}" == "{?}" - + @@ -19649,7 +19665,7 @@ b1! "operator<<( has_operator )" - + @@ -19662,7 +19678,7 @@ b1! "operator<<( has_template_operator )" - + @@ -19675,7 +19691,7 @@ b1! "{ StringMaker<has_maker> }" - + @@ -19688,7 +19704,7 @@ b1! "{ StringMaker<has_maker_and_operator> }" - + @@ -19701,7 +19717,7 @@ b1! "{ operator<<( has_operator ) }" - + @@ -19736,7 +19752,7 @@ b1! 4 == 4 - + @@ -19771,7 +19787,7 @@ b1! 6 == 6 - + @@ -19790,16 +19806,17 @@ b1! magic.tag == magic.tag - + - + + Why would you throw a std::string? - + @@ -19810,7 +19827,7 @@ b1! ""wide load"" == ""wide load"" - + @@ -19821,7 +19838,7 @@ b1! ""wide load"" == ""wide load"" - + @@ -19832,7 +19849,7 @@ b1! ""wide load"" == ""wide load"" - + @@ -19843,7 +19860,7 @@ b1! ""wide load"" == ""wide load"" - + @@ -19872,7 +19889,7 @@ b1! "Unknown enum value 10" - + @@ -19891,7 +19908,7 @@ b1! "1" == "1" - + @@ -19910,7 +19927,7 @@ b1! "E2{1}" == "E2{1}" - + @@ -19929,7 +19946,7 @@ b1! "1" == "1" - + @@ -19948,7 +19965,7 @@ b1! "{ }" == "{ }" - + @@ -19967,7 +19984,7 @@ b1! "{ 1.2f, 0 }" == "{ 1.2f, 0 }" - + @@ -19978,7 +19995,7 @@ b1! "{ 0 }" == "{ 0 }" - + @@ -19991,7 +20008,7 @@ b1! "{ "hello", "world" }" - + @@ -20004,7 +20021,7 @@ b1! "{ { 42 }, { }, 1.2f }" - + @@ -20039,7 +20056,7 @@ b1! 0.95 == 0.95 - +
@@ -20059,7 +20076,7 @@ b1! 0 == 0 - +
@@ -20103,9 +20120,9 @@ b1! 0 == 0 - +
- +
@@ -20157,9 +20174,9 @@ b1! 2 == 2 - +
- +
@@ -20178,7 +20195,7 @@ b1! 0 == 0 - +
@@ -20205,7 +20222,7 @@ b1! 1 == 1 - +
@@ -20232,7 +20249,7 @@ b1! 2 == 2 - +
@@ -20251,9 +20268,9 @@ b1! 1 == 1 - +
- + @@ -20274,7 +20291,7 @@ b1! "{ { "hello" }, { "world" } }" - + @@ -20301,7 +20318,7 @@ b1! "{ true, false }" == "{ true, false }" - + @@ -20328,7 +20345,7 @@ b1! "{ 42, 250 }" == "{ 42, 250 }" - + @@ -20355,7 +20372,7 @@ b1! "{ 42, 250 }" == "{ 42, 250 }" - + @@ -20384,7 +20401,7 @@ b1! "{ "hello", "world" }" - + @@ -20420,7 +20437,7 @@ b1! 10 >= 10 - + @@ -20464,9 +20481,9 @@ b1! 0 == 0 - + - + @@ -20501,7 +20518,7 @@ b1! 10 >= 10 - + @@ -20536,9 +20553,9 @@ b1! 5 >= 5 - + - + @@ -20557,7 +20574,7 @@ b1! 310016000 ns > 100 ms - + @@ -20584,17 +20601,17 @@ b1! 23.0 == 23.0 - +
- +
- +
- + - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index 1e54761035..08e7e6b473 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -1,7 +1,7 @@ - + @@ -20,7 +20,7 @@ 0 == 0 - + @@ -71,10 +71,10 @@ {?} >= {?} - + - + @@ -105,16 +105,16 @@ 0 == 0 - + - +
- +
- + @@ -125,7 +125,7 @@ [1403 helper] == [1403 helper] - + @@ -136,13 +136,13 @@ This info message starts with a linebreak This warning message starts with a linebreak - + 1514 - + This would not be caught previously @@ -160,7 +160,7 @@ Nor would this true - + @@ -187,7 +187,7 @@ Nor would this !false - +
@@ -215,7 +215,7 @@ Nor would this !false - +
@@ -226,9 +226,9 @@ Nor would this true - +
- + @@ -247,7 +247,7 @@ Nor would this 6 < 7 - + @@ -282,11 +282,11 @@ Nor would this 2 != 4 - +
- +
@@ -297,7 +297,7 @@ Nor would this 1 - +
@@ -308,7 +308,7 @@ Nor would this 2 - +
@@ -319,9 +319,9 @@ Nor would this 3 - +
- +
@@ -333,7 +333,7 @@ Nor would this 1 - +
@@ -351,7 +351,7 @@ Nor would this 3 - + @@ -378,11 +378,11 @@ Nor would this 3 - +
- +
i := 1 @@ -394,7 +394,7 @@ Nor would this k := 5
- +
i := 1 @@ -406,7 +406,7 @@ Nor would this k := 6
- +
i := 1 @@ -427,7 +427,7 @@ Nor would this k := 6
- +
i := 2 @@ -439,7 +439,7 @@ Nor would this k := 5
- +
i := 2 @@ -451,7 +451,7 @@ Nor would this k := 6
- +
i := 2 @@ -471,7 +471,7 @@ Nor would this k := 6 - + @@ -618,16 +618,16 @@ Nor would this 3 - + - + - + - + @@ -646,7 +646,7 @@ Nor would this 0.0 not is within 1 ULPs of -4.9406564584124654e-324 ([-9.8813129168249309e-324, -0.0000000000000000e+00]) - + @@ -665,7 +665,7 @@ Nor would this 0.0f not is within 1 ULPs of -1.40129846e-45f ([-2.80259693e-45, -0.00000000e+00]) - +
@@ -675,7 +675,7 @@ Nor would this expected exception - +
@@ -692,7 +692,7 @@ Nor would this expected exception - +
@@ -706,9 +706,9 @@ Nor would this thisThrows() - +
- + @@ -719,7 +719,7 @@ Nor would this 42 == {?} - + @@ -778,7 +778,7 @@ Nor would this true - + @@ -797,7 +797,7 @@ Nor would this 1 == 1 - + @@ -811,25 +811,25 @@ Nor would this {?} == 4 - +
- +
- +
- +
- +
- +
- + @@ -896,7 +896,7 @@ Nor would this !(1 == 1) - + @@ -963,7 +963,7 @@ Nor would this !(1 == 2) - +
@@ -983,7 +983,7 @@ Nor would this true == true - +
@@ -1002,7 +1002,7 @@ Nor would this false == false - +
@@ -1013,7 +1013,7 @@ Nor would this true - +
@@ -1024,7 +1024,7 @@ Nor would this true - +
@@ -1043,9 +1043,9 @@ Nor would this !false - +
- + @@ -1696,7 +1696,7 @@ Nor would this 3 < 9 - + @@ -1707,7 +1707,7 @@ Nor would this "hello" == "world" - + @@ -1718,7 +1718,7 @@ Nor would this "hello" == "hello" - + @@ -1729,7 +1729,7 @@ Nor would this 0 == 1 - + @@ -1740,7 +1740,7 @@ Nor would this 0 == 1 - + @@ -1751,7 +1751,7 @@ Nor would this 0 == 1 - + @@ -1762,7 +1762,7 @@ Nor would this 0 == 1 - + @@ -1773,7 +1773,7 @@ Nor would this 0 == 0 - + @@ -1784,7 +1784,7 @@ Nor would this 0 == 0 - + @@ -1795,7 +1795,7 @@ Nor would this 0 == 0 - + @@ -1806,7 +1806,7 @@ Nor would this 0 == 0 - + @@ -1817,7 +1817,7 @@ Nor would this 6 < 2 - + @@ -1828,7 +1828,7 @@ Nor would this 2 < 2 - + @@ -1839,7 +1839,7 @@ Nor would this 6 < 2 - + @@ -1850,7 +1850,7 @@ Nor would this 2 < 2 - + @@ -1861,7 +1861,7 @@ Nor would this 6 >= 2 - + @@ -1872,7 +1872,7 @@ Nor would this 2 >= 2 - + @@ -1883,7 +1883,7 @@ Nor would this 6 >= 2 - + @@ -1894,7 +1894,7 @@ Nor would this 2 >= 2 - + @@ -1905,7 +1905,7 @@ Nor would this 1.0 == 2 - + @@ -1916,7 +1916,7 @@ Nor would this 1.0f == 2 - + @@ -1927,7 +1927,7 @@ Nor would this 1 == 2 - + @@ -1938,7 +1938,7 @@ Nor would this 1.0 == 1 - + @@ -1949,7 +1949,7 @@ Nor would this 1.0f == 1 - + @@ -1960,7 +1960,7 @@ Nor would this 1 == 1 - + @@ -1971,7 +1971,7 @@ Nor would this 1 == 0 - + @@ -1982,7 +1982,7 @@ Nor would this 3 == 0 - + @@ -1993,7 +1993,7 @@ Nor would this 6 == 0 - + @@ -2004,7 +2004,7 @@ Nor would this 1 > 0 - + @@ -2015,7 +2015,7 @@ Nor would this 3 > 0 - + @@ -2026,7 +2026,7 @@ Nor would this 6 > 0 - + @@ -2037,7 +2037,7 @@ Nor would this 1 == 2 - + @@ -2048,7 +2048,7 @@ Nor would this 1 == 1 - + @@ -2059,7 +2059,7 @@ Nor would this 0 == 0 - + @@ -2070,7 +2070,7 @@ Nor would this 0 == 0 - + @@ -2081,7 +2081,7 @@ Nor would this 0 == 0 - + @@ -2092,7 +2092,7 @@ Nor would this 0 == 0 - + @@ -2103,7 +2103,7 @@ Nor would this 42 > 0 - + @@ -2114,7 +2114,7 @@ Nor would this 9 > 0 - + @@ -2125,7 +2125,7 @@ Nor would this 42 > 0 - + @@ -2136,7 +2136,7 @@ Nor would this 9 > 0 - + @@ -2187,19 +2187,19 @@ Nor would this 1.23 == Approx( 1.0 ) - +
- +
- +
to infinity and beyond - + @@ -2218,7 +2218,7 @@ Nor would this {?} == {?} - + @@ -2269,10 +2269,10 @@ Nor would this 100.3 == Approx( 100.0 ) - + - + @@ -2291,7 +2291,7 @@ Nor would this 8 == 8 - + @@ -2313,10 +2313,10 @@ Nor would this unexpected exception - + - + @@ -2375,7 +2375,7 @@ Nor would this Approx(0).epsilon(1.0001), std::domain_error - + @@ -2418,7 +2418,7 @@ Nor would this 245.5f == Approx( 245.25 ) - + @@ -2437,7 +2437,7 @@ Nor would this 3.1428571429 != Approx( 3.141 ) - + @@ -2456,7 +2456,7 @@ Nor would this 1.23 == Approx( 1.231 ) - + @@ -2475,7 +2475,7 @@ Nor would this 0.0f == Approx( 0.0 ) - + @@ -2494,7 +2494,7 @@ Nor would this 0 == Approx( 0.0 ) - + @@ -2537,7 +2537,7 @@ Nor would this 1.234 == Approx( 1.2339999676 ) - +
@@ -2557,7 +2557,7 @@ Nor would this 1 not matches predicate: "always false" - +
@@ -2576,9 +2576,9 @@ Nor would this "This wouldn't pass" not matches undescribed predicate - +
- + @@ -2621,7 +2621,7 @@ Nor would this !(Val: 1 ^ Val: 1) - + @@ -2650,9 +2650,9 @@ Nor would this true - + - + @@ -2680,11 +2680,11 @@ Nor would this true - + - + - +
@@ -2712,7 +2712,7 @@ Nor would this { 4, 5, 6 } not contains element 1 - +
@@ -2739,7 +2739,7 @@ Nor would this { 4, 5, 6 } not contains element 0 - +
@@ -2750,7 +2750,7 @@ Nor would this { "abc", "abcd", "abcde" } contains element 4 - +
@@ -2769,7 +2769,7 @@ Nor would this { 1, 2, 3, 4, 5 } not contains element 8 - +
@@ -2788,7 +2788,7 @@ Nor would this { 1, 2, 3 } not contains element 9 - +
@@ -2799,9 +2799,9 @@ Nor would this { 1.0, 2.0, 3.0, 0.0 } contains element matching is within 0.5 of 0.5 - +
- +
@@ -2853,7 +2853,7 @@ Nor would this { } is empty - +
@@ -2864,7 +2864,7 @@ Nor would this {?} not is empty - +
@@ -2875,9 +2875,9 @@ Nor would this {?} is empty - +
- + @@ -2901,7 +2901,7 @@ Nor would this a == 1 := true - + @@ -2925,7 +2925,7 @@ Nor would this (2, 3) := 3 - + @@ -2961,7 +2961,7 @@ Nor would this '{' := '{' - +
@@ -2976,7 +2976,7 @@ Nor would this true - +
@@ -2990,9 +2990,9 @@ Nor would this true - +
- +
@@ -3012,7 +3012,7 @@ Nor would this !false - +
@@ -3063,9 +3063,9 @@ Nor would this !false - +
- +
@@ -3093,7 +3093,7 @@ Nor would this !false - +
@@ -3128,9 +3128,9 @@ Nor would this true - +
- +
@@ -3166,7 +3166,7 @@ Nor would this '\f' == '\f' - +
@@ -3209,7 +3209,7 @@ Nor would this 'Z' == 'Z' - +
@@ -3252,9 +3252,9 @@ Nor would this 5 == 5 - +
- + @@ -3273,7 +3273,7 @@ Nor would this "foo" == "foo" - +
@@ -3285,7 +3285,7 @@ Nor would this !{?} - +
@@ -3304,9 +3304,9 @@ Nor would this { "aaa", "bbb" } == { "aaa", "bbb" } - +
- +
@@ -3318,7 +3318,7 @@ Nor would this true - +
@@ -3341,7 +3341,7 @@ Using code: 0 " - +
@@ -3364,9 +3364,9 @@ C " - +
- + @@ -3393,7 +3393,7 @@ C 1 ( equals: (int) 1 or (string) "1" and equals: (long long) 1 and equals: (T) 1 and equals: true ) - + @@ -3420,7 +3420,7 @@ C 1 ( equals: (int) 1 or (string) "1" or equals: (long long) 1 or equals: (T) 1 or equals: true ) - + @@ -3455,10 +3455,10 @@ C 1 equals: (int) 1 or (string) "1" - + - + @@ -3485,7 +3485,7 @@ C 1 ( equals: (int) 1 or (string) "1" or not equals: (long long) 1 ) - + @@ -3544,7 +3544,7 @@ C "foobar" ( ( starts with: "foo" and ends with: "bar" ) or Equals: { 'o', 'o', 'f', 'b', 'a', 'r' } ) - + @@ -3555,7 +3555,7 @@ C { 1, 2, 3 } ( Equals: { 1, 2, 3 } or Equals: { 0, 1, 2 } or Equals: { 4, 5, 6 } ) - + @@ -3654,7 +3654,7 @@ C { 1, 2 } == { 1, 2 } - + @@ -3673,7 +3673,7 @@ C 0x == 0x - + @@ -3708,7 +3708,7 @@ C !({?} != {?}) - + @@ -3807,7 +3807,7 @@ C Approx( 11.0 ) >= StrongDoubleTypedef(10) - + @@ -3818,7 +3818,7 @@ C 54 == 54 - + @@ -3869,7 +3869,7 @@ C -2147483648 > 2 - + @@ -3976,7 +3976,7 @@ C 4294967295 (0x) > 4 - +
@@ -4004,7 +4004,7 @@ C true - +
@@ -4031,9 +4031,9 @@ C true - +
- +
@@ -4061,7 +4061,7 @@ C true - +
@@ -4088,9 +4088,9 @@ C true - +
- + @@ -4109,7 +4109,7 @@ C "this string contains 'abc' as a substring" contains: "STRING" - +
@@ -4121,7 +4121,7 @@ C 1 == 1 - +
@@ -4132,7 +4132,7 @@ C 1 == 1 - +
@@ -4143,7 +4143,7 @@ C 1 == 1 - +
@@ -4154,7 +4154,7 @@ C 1 == 1 - +
@@ -4165,7 +4165,7 @@ C 1 == 1 - +
@@ -4176,7 +4176,7 @@ C 1 == 1 - +
@@ -4187,7 +4187,7 @@ C 1 == 1 - +
@@ -4198,7 +4198,7 @@ C 1 == 1 - +
@@ -4209,7 +4209,7 @@ C 1 == 1 - +
@@ -4220,7 +4220,7 @@ C 1 == 1 - +
@@ -4231,7 +4231,7 @@ C 1 == 1 - +
@@ -4242,7 +4242,7 @@ C 1 == 1 - +
@@ -4261,9 +4261,9 @@ C 6 == 6 - +
- + @@ -4274,7 +4274,7 @@ C true - + @@ -4288,7 +4288,7 @@ C custom exception - not std - + @@ -4302,13 +4302,13 @@ C custom exception - not std - + custom std exception - + @@ -4327,7 +4327,7 @@ C 0.00001 != Approx( 0.0000001 ) - + @@ -4356,7 +4356,7 @@ C "{** unexpected enum value **}" - + @@ -4367,7 +4367,7 @@ C true - + @@ -4378,7 +4378,7 @@ C Catch::TestCaseInfo( "", { "fake test name", "[]" }, dummySourceLineInfo ) - + @@ -4397,7 +4397,7 @@ C "this string contains 'abc' as a substring" ends with: "this" (case insensitive) - + @@ -4442,7 +4442,7 @@ C "Value2" == "Value2" - + @@ -4461,7 +4461,7 @@ C "Blue" == "Blue" - + @@ -4472,7 +4472,7 @@ C 101.01 != Approx( 100.0 ) - + @@ -4579,7 +4579,7 @@ C 1.3 == Approx( 1.301 ) - + @@ -4638,7 +4638,7 @@ C 1.3 == Approx( 1.3 ) - + @@ -4657,7 +4657,7 @@ C "this string contains 'abc' as a substring" equals: "this string contains 'abc' as a substring" (case insensitive) - + @@ -4676,7 +4676,7 @@ C "this string contains 'abc' as a substring" equals: "something else" (case insensitive) - + @@ -4707,7 +4707,7 @@ C "StringMakerException" - +
@@ -4727,7 +4727,7 @@ C doesNotThrow(), SpecialException, ExceptionMatcher{ 1 } - +
@@ -4752,7 +4752,7 @@ C Unknown exception - +
@@ -4771,9 +4771,9 @@ C SpecialException::what special exception has value of 1 - +
- + @@ -4792,7 +4792,7 @@ C SpecialException::what special exception has value of 2 - + @@ -4827,7 +4827,7 @@ C SpecialException::what matches "starts with: "Special"" - +
@@ -4839,7 +4839,7 @@ C "expected exception" equals: "expected exception" - +
@@ -4850,7 +4850,7 @@ C "expected exception" equals: "expected exception" (case insensitive) - +
@@ -4885,9 +4885,9 @@ C "expected exception" contains: "except" (case insensitive) - +
- + @@ -4922,7 +4922,7 @@ C SpecialException::what exception message matches "SpecialException::what" - + @@ -4955,17 +4955,17 @@ C expected exception - + This is a failure - + - + @@ -4974,7 +4974,7 @@ C This message appears in the output - + @@ -5017,7 +5017,7 @@ C 3628800 (0x) == 3628800 (0x) - +
@@ -5062,9 +5062,9 @@ C 0.0 and 2.22507e-308 are within 2.22045e-12% of each other - +
- +
@@ -5131,7 +5131,7 @@ C -10.0 is within 0.5 of -9.6 - +
@@ -5190,7 +5190,7 @@ C -0.0 is within 0 ULPs of 0.0000000000000000e+00 ([0.0000000000000000e+00, 0.0000000000000000e+00]) - +
@@ -5217,7 +5217,7 @@ C 0.0001 ( is within 0.001 of 0.0 or and 0 are within 10% of each other ) - +
@@ -5268,9 +5268,9 @@ C WithinRel( 1., 1. ), std::domain_error - +
- +
@@ -5315,9 +5315,9 @@ C 0.0f and 1.17549e-38 are within 0.00119209% of each other - +
- +
@@ -5392,7 +5392,7 @@ C -10.0f is within 0.5 of -9.6000003815 - +
@@ -5459,7 +5459,7 @@ C -0.0f is within 0 ULPs of 0.00000000e+00f ([0.00000000e+00, 0.00000000e+00]) - +
@@ -5486,7 +5486,7 @@ C 0.0001f ( is within 0.001 of 0.0 or and 0 are within 10% of each other ) - +
@@ -5545,9 +5545,9 @@ C WithinRel( 1.f, 1.f ), std::domain_error - +
- +
@@ -5560,9 +5560,9 @@ C 0 == 0 - +
- +
@@ -5574,9 +5574,9 @@ C 0 == 0 - +
- +
@@ -5588,9 +5588,9 @@ C 0 == 0 - +
- +
@@ -5602,9 +5602,9 @@ C filter([] (int) {return false; }, value(1)), Catch::GeneratorException - +
- +
@@ -5615,7 +5615,7 @@ C 1 < 4 - +
@@ -5626,7 +5626,7 @@ C 2 < 4 - +
@@ -5637,7 +5637,7 @@ C 3 < 4 - +
@@ -5649,9 +5649,9 @@ C 0 == 0 - +
- +
@@ -5663,9 +5663,9 @@ C 0 == 0 - +
- +
@@ -5677,9 +5677,9 @@ C 0 == 0 - +
- +
@@ -5691,9 +5691,9 @@ C 1 == 1 - +
- +
@@ -5705,9 +5705,9 @@ C 1 == 1 - +
- +
@@ -5719,9 +5719,9 @@ C 1 == 1 - +
- +
@@ -5733,9 +5733,9 @@ C 1 == 1 - +
- +
@@ -5747,9 +5747,9 @@ C 1 == 1 - +
- +
@@ -5761,9 +5761,9 @@ C 1 == 1 - +
- +
@@ -5774,7 +5774,7 @@ C 1 > 0 - +
@@ -5785,7 +5785,7 @@ C 2 > 0 - +
@@ -5796,7 +5796,7 @@ C 3 > 0 - +
@@ -5807,7 +5807,7 @@ C 1 > 0 - +
@@ -5818,7 +5818,7 @@ C 2 > 0 - +
@@ -5829,7 +5829,7 @@ C 3 > 0 - +
@@ -5849,9 +5849,9 @@ C 1 == 1 - +
- +
@@ -5871,9 +5871,9 @@ C 2 == 2 - +
- +
@@ -5893,9 +5893,9 @@ C 3 == 3 - +
- +
@@ -5923,9 +5923,9 @@ C 1 < 3 - +
- +
@@ -5953,9 +5953,9 @@ C 2 < 3 - +
- +
@@ -5967,9 +5967,9 @@ C 0 == 0 - +
- +
@@ -5981,9 +5981,9 @@ C 0 == 0 - +
- +
@@ -5995,9 +5995,9 @@ C 0 == 0 - +
- +
@@ -6009,11 +6009,11 @@ C chunk(2, value(1)), Catch::GeneratorException - +
- +
- +
@@ -6025,7 +6025,7 @@ C -3 < 1 - +
@@ -6036,7 +6036,7 @@ C -2 < 1 - +
@@ -6047,7 +6047,7 @@ C -1 < 1 - +
@@ -6058,7 +6058,7 @@ C 4 > 1 - +
@@ -6069,7 +6069,7 @@ C 4 > 2 - +
@@ -6080,7 +6080,7 @@ C 4 > 3 - +
@@ -6091,7 +6091,7 @@ C -3 < 2 - +
@@ -6102,7 +6102,7 @@ C -2 < 2 - +
@@ -6113,7 +6113,7 @@ C -1 < 2 - +
@@ -6124,7 +6124,7 @@ C 8 > 1 - +
@@ -6135,7 +6135,7 @@ C 8 > 2 - +
@@ -6146,7 +6146,7 @@ C 8 > 3 - +
@@ -6157,7 +6157,7 @@ C -3 < 3 - +
@@ -6168,7 +6168,7 @@ C -2 < 3 - +
@@ -6179,7 +6179,7 @@ C -1 < 3 - +
@@ -6190,7 +6190,7 @@ C 12 > 1 - +
@@ -6201,7 +6201,7 @@ C 12 > 2 - +
@@ -6212,9 +6212,9 @@ C 12 > 3 - +
- +
@@ -6234,7 +6234,7 @@ C !false - +
@@ -6285,7 +6285,7 @@ C !false - +
@@ -6368,7 +6368,7 @@ C !false - +
@@ -6427,7 +6427,7 @@ C !false - +
@@ -6463,9 +6463,9 @@ C !false - +
- +
@@ -6501,9 +6501,9 @@ C !false - +
- +
@@ -6523,9 +6523,9 @@ C filter([](int) { return false; }, values({ 1, 2, 3 })), Catch::GeneratorException - +
- +
@@ -6561,9 +6561,9 @@ C !false - +
- +
@@ -6583,9 +6583,9 @@ C !false - +
- +
@@ -6636,7 +6636,7 @@ C !false - +
@@ -6687,7 +6687,7 @@ C !false - +
@@ -6707,9 +6707,9 @@ C !false - +
- +
@@ -6809,9 +6809,9 @@ C !false - +
- +
@@ -6880,11 +6880,11 @@ C !false - +
- +
- +
@@ -6953,11 +6953,11 @@ C !false - +
- +
- +
@@ -7027,13 +7027,13 @@ C !false - +
- +
- + - +
@@ -7103,13 +7103,13 @@ C !false - +
- +
- + - +
@@ -7195,13 +7195,13 @@ C !false - +
- +
- + - +
@@ -7663,13 +7663,13 @@ C !false - +
- +
- + - +
@@ -7815,13 +7815,13 @@ C !false - +
- +
- + - +
@@ -7967,13 +7967,13 @@ C !false - +
- +
- + - +
@@ -8043,13 +8043,13 @@ C !false - +
- +
- + - +
@@ -8119,13 +8119,13 @@ C !false - +
- +
- + - +
@@ -8211,15 +8211,15 @@ C !false - +
- +
- + - + - + @@ -8254,7 +8254,7 @@ C 1.23 >= Approx( 1.24 ) - + @@ -8267,7 +8267,7 @@ C 130711275 (0x) - + @@ -8280,7 +8280,7 @@ C 3422778688 (0x) - +
@@ -8294,7 +8294,7 @@ C 2668622104 (0x) - +
@@ -8307,7 +8307,7 @@ C 3916075712 (0x) - +
@@ -8320,9 +8320,9 @@ C 3429949824 (0x) - +
- + @@ -8335,7 +8335,7 @@ C 3422778688 (0x) - + @@ -8344,7 +8344,7 @@ C this is a warning - + @@ -8361,7 +8361,7 @@ C 2 == 1 - + @@ -8426,7 +8426,7 @@ C 2 == 2 - + @@ -8583,7 +8583,7 @@ C 10 < 10 - + @@ -8626,7 +8626,7 @@ C 5 != 5 - + @@ -8717,7 +8717,7 @@ C 5 != 6 - + @@ -8728,7 +8728,7 @@ C true - + @@ -8763,10 +8763,10 @@ C 1.23 <= Approx( 1.22 ) - + - + @@ -8777,7 +8777,7 @@ C "this string contains 'abc' as a substring" ( contains: "string" and contains: "abc" and contains: "substring" and contains: "contains" ) - + @@ -8796,7 +8796,7 @@ C "some completely different text that contains one common word" ( contains: "string" or contains: "different" or contains: "random" ) - + @@ -8807,7 +8807,7 @@ C "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "substring" ) - + @@ -8818,7 +8818,7 @@ C "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "random" ) - + @@ -8829,7 +8829,7 @@ C "this string contains 'abc' as a substring" not contains: "different" - + @@ -8840,44 +8840,44 @@ C "this string contains 'abc' as a substring" not contains: "substring" - +
- +
- +
- +
- +
- +
- +
- +
- +
- +
- +
- + @@ -8896,7 +8896,7 @@ C "expected exception" equals: "should fail" - + @@ -8909,7 +8909,7 @@ C { "Hello", "world", "Goodbye", "world" } - + @@ -8977,7 +8977,7 @@ C true == true - + @@ -9044,9 +9044,9 @@ C true == true - + - + @@ -9177,19 +9177,19 @@ C 99 > -6 - + This one ran - + custom exception - + @@ -9216,10 +9216,10 @@ C !{?} - + - + @@ -9374,7 +9374,7 @@ C "hello" <= "a" - + @@ -9513,7 +9513,7 @@ C "hello" > "a" - +
@@ -9567,7 +9567,7 @@ C 1827115164 (0x) - +
@@ -9670,24 +9670,24 @@ C 4261393167 (0x) - +
- +
Message from section one - +
Message from section two - +
- + @@ -9722,7 +9722,7 @@ C ( EvilMatcher() && EvilMatcher() ) || !EvilMatcher() - +
@@ -9758,7 +9758,7 @@ C {?} == {?} - +
@@ -9817,9 +9817,9 @@ C !{?} - +
- + @@ -9846,7 +9846,7 @@ C true - +
@@ -9866,7 +9866,7 @@ C 8 == 8 - +
@@ -9885,7 +9885,7 @@ C "Could not parse '-1' as shard count" contains: "Could not parse '-1' as shard count" - +
@@ -9904,7 +9904,7 @@ C "Shard count must be positive" contains: "Shard count must be positive" - +
@@ -9923,7 +9923,7 @@ C 2 == 2 - +
@@ -9942,7 +9942,7 @@ C "Could not parse '-12' as shard index" contains: "Could not parse '-12' as shard index" - +
@@ -9961,9 +9961,9 @@ C 0 == 0 - +
- + @@ -10032,7 +10032,7 @@ C true - +
@@ -10052,7 +10052,7 @@ C 1 == 1 - +
@@ -10063,7 +10063,7 @@ C !{?} - +
@@ -10082,9 +10082,9 @@ C 3 == 3 - +
- + @@ -10151,7 +10151,7 @@ C 0 != 0x - +
@@ -10171,7 +10171,7 @@ C 13 == 13 - +
@@ -10190,9 +10190,9 @@ C 17 == 17 - +
- + @@ -10203,7 +10203,7 @@ C "foo" matches undescribed predicate - +
@@ -10223,7 +10223,7 @@ C "" == "" - +
@@ -10314,7 +10314,7 @@ C {?} == {?} - +
@@ -10350,9 +10350,9 @@ C true - +
- +
@@ -10388,9 +10388,9 @@ C true - +
- +
@@ -10426,9 +10426,9 @@ C true - +
- +
@@ -10454,9 +10454,9 @@ C { {?} } == { {?} } - +
- +
@@ -10482,9 +10482,9 @@ C { {?} } == { {?} } - +
- +
@@ -10510,9 +10510,9 @@ C { {?} } == { {?} } - +
- +
@@ -10532,9 +10532,9 @@ C "Unrecognized reporter, 'unsupported'. Check available with --list-reporters" contains: "Unrecognized reporter" - +
- +
@@ -10560,9 +10560,9 @@ C { {?} } == { {?} } - +
- +
@@ -10588,9 +10588,9 @@ C { {?} } == { {?} } - +
- +
@@ -10611,11 +10611,11 @@ C { {?}, {?} } == { {?}, {?} } - +
- +
- +
@@ -10636,11 +10636,11 @@ C { {?}, {?} } == { {?}, {?} } - +
- +
- +
@@ -10661,11 +10661,11 @@ C "Only one reporter may have unspecified output file." contains: "Only one reporter may have unspecified output file." - +
- +
- +
@@ -10685,9 +10685,9 @@ C true == true - +
- +
@@ -10707,9 +10707,9 @@ C true - +
- +
@@ -10729,9 +10729,9 @@ C 1 == 1 - +
- +
@@ -10751,9 +10751,9 @@ C 2 == 2 - +
- +
@@ -10773,9 +10773,9 @@ C "Unable to convert 'oops' to destination type" ( contains: "convert" and contains: "oops" ) - +
- +
@@ -10796,11 +10796,11 @@ C 0 == 0 - +
- +
- +
@@ -10821,11 +10821,11 @@ C 1 == 1 - +
- +
- +
@@ -10846,11 +10846,11 @@ C 2 == 2 - +
- +
- +
@@ -10871,11 +10871,11 @@ C 3 == 3 - +
- +
- +
@@ -10896,11 +10896,11 @@ C "keypress argument must be one of: never, start, exit or both. 'sometimes' not recognised" ( contains: "never" and contains: "both" ) - +
- +
- +
@@ -10920,9 +10920,9 @@ C true - +
- +
@@ -10942,9 +10942,9 @@ C true - +
- +
@@ -10964,9 +10964,9 @@ C "filename.ext" == "filename.ext" - +
- +
@@ -10986,9 +10986,9 @@ C "filename.ext" == "filename.ext" - +
- +
@@ -11024,9 +11024,9 @@ C true == true - +
- +
@@ -11046,9 +11046,9 @@ C 0 == 0 - +
- +
@@ -11068,9 +11068,9 @@ C 0 == 0 - +
- +
@@ -11090,9 +11090,9 @@ C 1 == 1 - +
- +
@@ -11112,9 +11112,9 @@ C 3 == 3 - +
- +
@@ -11134,9 +11134,9 @@ C "colour mode must be one of: default, ansi, win32, or none. 'wrong' is not recognised" contains: "colour mode must be one of" - +
- +
@@ -11156,9 +11156,9 @@ C 200 == 200 - +
- +
@@ -11178,9 +11178,9 @@ C 20000 (0x) == 20000 (0x) - +
- +
@@ -11200,9 +11200,9 @@ C 0.99 == Approx( 0.99 ) - +
- +
@@ -11222,9 +11222,9 @@ C true - +
- +
@@ -11244,11 +11244,11 @@ C 10 == 10 - +
- +
- + @@ -11259,7 +11259,7 @@ C 3 >= 1 - + @@ -11270,7 +11270,7 @@ C 2 >= 1 - + @@ -11281,7 +11281,7 @@ C 1 >= 1 - + @@ -11308,7 +11308,7 @@ C Catch::generateRandomSeed(method) - + @@ -11319,7 +11319,7 @@ C Catch::generateRandomSeed(static_cast<Catch::GenerateFrom>(77)) - + @@ -11330,7 +11330,7 @@ C "{ }" == "{ }" - + @@ -11341,7 +11341,7 @@ C Hey, its truthy! - + @@ -11368,7 +11368,7 @@ C "this string contains 'abc' as a substring" matches "this string contains 'abc' as a" case sensitively - + @@ -11379,7 +11379,7 @@ C "'::' is not allowed in reporter name: 'with::doublecolons'" equals: "'::' is not allowed in reporter name: 'with::doublecolons'" - + @@ -11390,7 +11390,7 @@ C { 'a', 'b' } not UnorderedEquals: { 'c', 'b' } - + @@ -11417,7 +11417,7 @@ C " contains: "fakeTag" - + @@ -11442,7 +11442,7 @@ C " contains: "fake reporter" - + @@ -11469,7 +11469,7 @@ C " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11495,7 +11495,7 @@ C " contains: "fakeTag" - + @@ -11520,7 +11520,7 @@ C " contains: "fake reporter" - + @@ -11547,7 +11547,7 @@ C " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11573,7 +11573,7 @@ C " contains: "fakeTag" - + @@ -11598,7 +11598,7 @@ C " contains: "fake reporter" - + @@ -11625,7 +11625,7 @@ C " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11652,7 +11652,7 @@ All available tags: " contains: "fakeTag" - + @@ -11678,7 +11678,7 @@ Available reporters: " contains: "fake reporter" - + @@ -11706,7 +11706,7 @@ All available test cases: " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11733,7 +11733,7 @@ All available tags: " contains: "fakeTag" - + @@ -11759,7 +11759,7 @@ Available reporters: " contains: "fake reporter" - + @@ -11787,7 +11787,7 @@ All available test cases: " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11813,7 +11813,7 @@ All available test cases: " contains: "fakeTag" - + @@ -11838,7 +11838,7 @@ All available test cases: " contains: "fake reporter" - + @@ -11865,7 +11865,7 @@ All available test cases: " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11891,7 +11891,7 @@ All available test cases: " contains: "fakeTag" - + @@ -11916,7 +11916,7 @@ All available test cases: " contains: "fake reporter" - + @@ -11943,7 +11943,7 @@ All available test cases: " ( contains: "fake test name" and contains: "fakeTestTag" ) - + @@ -11973,7 +11973,7 @@ All available test cases: </TagsFromMatchingTests>" contains: "fakeTag" - + @@ -12001,7 +12001,7 @@ All available test cases: </AvailableReporters>" contains: "fake reporter" - + @@ -12034,18 +12034,18 @@ All available test cases: </MatchingTests>" ( contains: "fake test name" and contains: "fakeTestTag" ) - + - + - + - + - +
@@ -12067,13 +12067,13 @@ All available test cases: 1 > 0 - +
- + - + - +
@@ -12097,29 +12097,29 @@ All available test cases: true - +
- + - + - + - + - +
- +
- +
- +
- +
@@ -12167,15 +12167,15 @@ All available test cases: 10 >= 10 - +
- + - + - + - +
@@ -12204,16 +12204,16 @@ All available test cases: 0 == 0 - +
- + - + - + - + A string sent directly to stdout @@ -12288,16 +12288,16 @@ A string sent to stderr via clog Approx( 1.23 ) != 1.24 - +
- +
- +
- + Message from section one Message from section two @@ -12321,7 +12321,7 @@ Message from section two "this string contains 'abc' as a substring" starts with: "string" (case insensitive) - +
@@ -12333,7 +12333,7 @@ Message from section two "{ 1 }" == "{ 1 }" - +
@@ -12344,7 +12344,7 @@ Message from section two "{ 3, 2, 1 }" == "{ 3, 2, 1 }" - +
@@ -12357,9 +12357,9 @@ Message from section two "{ { "1:1", "1:2", "1:3" }, { "2:1", "2:2" } }" - +
- + @@ -12426,7 +12426,7 @@ Message from section two "this string contains 'abc' as a substring" ends with: " substring" (case insensitive) - +
@@ -12454,7 +12454,7 @@ Message from section two 0 == 0 - +
@@ -12489,7 +12489,7 @@ Message from section two "hello" == "hello" - +
@@ -12508,7 +12508,7 @@ Message from section two original.data() - +
@@ -12519,7 +12519,7 @@ Message from section two "original string" == "original string" - +
@@ -12530,7 +12530,7 @@ Message from section two "original string" == "original string" - +
@@ -12566,9 +12566,9 @@ Message from section two hello == "hello" - +
- +
@@ -12588,9 +12588,9 @@ Message from section two 0 == 0 - +
- +
@@ -12602,9 +12602,9 @@ Message from section two "hello world!" == "hello world!" - +
- +
@@ -12616,9 +12616,9 @@ Message from section two "hello world!" == "hello world!" - +
- +
@@ -12630,9 +12630,9 @@ Message from section two true - +
- +
@@ -12644,9 +12644,9 @@ Message from section two 0 == 0 - +
- +
@@ -12658,9 +12658,9 @@ Message from section two true - +
- +
@@ -12687,7 +12687,7 @@ Message from section two Hello != Hel - +
@@ -12707,9 +12707,9 @@ Message from section two 17 == 17 - +
- +
@@ -12729,9 +12729,9 @@ Message from section two 17 == 17 - +
- +
@@ -12751,9 +12751,9 @@ Message from section two 17 == 17 - +
- +
@@ -12773,9 +12773,9 @@ Message from section two 11 == 11 - +
- +
@@ -12795,9 +12795,9 @@ Message from section two 11 == 11 - +
- +
@@ -12810,7 +12810,7 @@ Message from section two "some string += the stringref contents" - +
@@ -12821,18 +12821,18 @@ Message from section two "abrakadabra" == "abrakadabra" - +
- +
- +
- +
- + @@ -12851,7 +12851,7 @@ Message from section two ""abc"" == ""abc"" - + @@ -12870,7 +12870,7 @@ Message from section two ""abc"" == ""abc"" - + @@ -12889,7 +12889,7 @@ Message from section two ""abc"" == ""abc"" - + @@ -12924,7 +12924,7 @@ Message from section two 1 ns != 1 us - + @@ -12943,7 +12943,7 @@ Message from section two 1 ps != 1 as - + @@ -12956,7 +12956,7 @@ Message from section two {iso8601-timestamp} - + @@ -12974,7 +12974,7 @@ Message from section two " - +
@@ -13018,7 +13018,7 @@ Message from section two Redefined at: file:10" contains: "10" - +
@@ -13053,9 +13053,9 @@ Message from section two registry.add( "[@no square bracket at end", "", Catch::SourceLineInfo( "file", 3 ) ) - +
- + @@ -13074,7 +13074,7 @@ Message from section two { {?}, {?} } ( Contains: {?} and Contains: {?} ) - + @@ -13085,7 +13085,7 @@ Message from section two 1 == 1 - + @@ -13096,7 +13096,7 @@ Message from section two 1 == 1 - + @@ -13107,7 +13107,7 @@ Message from section two 1.0 == 1 - + @@ -13118,7 +13118,7 @@ Message from section two 1 > 0 - + @@ -13129,7 +13129,7 @@ Message from section two 4 > 0 - + @@ -13140,7 +13140,7 @@ Message from section two 1 > 0 - + @@ -13151,7 +13151,7 @@ Message from section two 4 > 0 - + @@ -13162,7 +13162,7 @@ Message from section two 4 > 0 - + @@ -13173,7 +13173,7 @@ Message from section two 1 > 0 - + @@ -13184,7 +13184,7 @@ Message from section two 4 > 0 - + @@ -13220,7 +13220,7 @@ Message from section two 10 >= 10 - + @@ -13264,9 +13264,9 @@ Message from section two 0 == 0 - + - + @@ -13301,7 +13301,7 @@ Message from section two 10 >= 10 - + @@ -13336,9 +13336,9 @@ Message from section two 5 >= 5 - + - + @@ -13374,7 +13374,7 @@ Message from section two 10 >= 10 - + @@ -13418,9 +13418,9 @@ Message from section two 0 == 0 - + - + @@ -13455,7 +13455,7 @@ Message from section two 10 >= 10 - + @@ -13490,9 +13490,9 @@ Message from section two 5 >= 5 - + - + @@ -13528,7 +13528,7 @@ Message from section two 10 >= 10 - + @@ -13572,9 +13572,9 @@ Message from section two 0 == 0 - + - + @@ -13609,7 +13609,7 @@ Message from section two 10 >= 10 - + @@ -13644,9 +13644,9 @@ Message from section two 5 >= 5 - + - + @@ -13682,7 +13682,7 @@ Message from section two 10 >= 10 - + @@ -13726,9 +13726,9 @@ Message from section two 0 == 0 - + - + @@ -13763,7 +13763,7 @@ Message from section two 10 >= 10 - + @@ -13798,9 +13798,9 @@ Message from section two 5 >= 5 - + - + @@ -13836,7 +13836,7 @@ Message from section two 12 >= 12 - + @@ -13880,9 +13880,9 @@ Message from section two 0 == 0 - + - + @@ -13917,7 +13917,7 @@ Message from section two 12 >= 12 - + @@ -13952,9 +13952,9 @@ Message from section two 6 >= 6 - + - + @@ -13990,7 +13990,7 @@ Message from section two 8 >= 8 - + @@ -14034,9 +14034,9 @@ Message from section two 0 == 0 - + - + @@ -14071,7 +14071,7 @@ Message from section two 8 >= 8 - + @@ -14106,9 +14106,9 @@ Message from section two 4 >= 4 - + - + @@ -14144,7 +14144,7 @@ Message from section two 10 >= 10 - + @@ -14188,9 +14188,9 @@ Message from section two 0 == 0 - + - + @@ -14225,7 +14225,7 @@ Message from section two 10 >= 10 - + @@ -14260,9 +14260,9 @@ Message from section two 5 >= 5 - + - + @@ -14298,7 +14298,7 @@ Message from section two 30 >= 30 - + @@ -14342,9 +14342,9 @@ Message from section two 0 == 0 - + - + @@ -14379,7 +14379,7 @@ Message from section two 30 >= 30 - + @@ -14414,9 +14414,9 @@ Message from section two 15 >= 15 - + - + @@ -14435,10 +14435,10 @@ Message from section two {?} == {?} - + - + @@ -14449,10 +14449,10 @@ Message from section two 3221225472 (0x) == 3221225472 - + - + @@ -14487,7 +14487,7 @@ Message from section two false - + @@ -14499,7 +14499,7 @@ Message from section two - + @@ -14511,7 +14511,7 @@ Message from section two - + @@ -14522,7 +14522,7 @@ Message from section two 1 == 2 - +
@@ -14538,7 +14538,7 @@ Message from section two " contains: "[fakeTag]" - +
@@ -14552,7 +14552,7 @@ Message from section two " ( contains: "fake reporter" and contains: "fake description" ) - +
@@ -14568,7 +14568,7 @@ Message from section two " ( contains: "fake test name" and contains: "fakeTestTag" ) - +
@@ -14582,18 +14582,18 @@ Message from section two " ( contains: "fakeListener" and contains: "fake description" ) - +
- + - + For some reason someone is throwing a string literal! - + @@ -14645,7 +14645,7 @@ Message from section two true - + @@ -14745,9 +14745,9 @@ Message from section two true - + - + @@ -14855,9 +14855,9 @@ Message from section two true - + - + @@ -14958,11 +14958,11 @@ Message from section two true - + - + - + @@ -15095,11 +15095,11 @@ Message from section two true - + - + - + @@ -15166,9 +15166,9 @@ Message from section two true - + - + @@ -15251,13 +15251,13 @@ There is no extra whitespace here There is no extra whitespace here - + 3.14 - +
@@ -15269,7 +15269,7 @@ There is no extra whitespace here 3 == 3 - +
@@ -15280,9 +15280,9 @@ There is no extra whitespace here 3 == 3 - +
- +
@@ -15302,7 +15302,7 @@ There is no extra whitespace here { { 0, 1, 2, 3, 5 }, { 4, -3, -2, 5, 0 }, { 0, 0, 0, 5, 0 }, { 0, -5, 0, 5, 0 }, { 1, 0, 0, -1, 5 } } not all match ( contains element 0 and contains element 1 ) - +
@@ -15313,7 +15313,7 @@ There is no extra whitespace here { 1, 2, 3, 4, 5 } all match matches undescribed predicate - +
@@ -15365,9 +15365,9 @@ There is no extra whitespace here true - +
- +
@@ -15419,11 +15419,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -15436,9 +15436,9 @@ There is no extra whitespace here { true, true, true, true, true } contains only true - +
- +
@@ -15450,9 +15450,9 @@ There is no extra whitespace here { } contains only true - +
- +
@@ -15464,9 +15464,9 @@ There is no extra whitespace here { true, true, false, true, true } not contains only true - +
- +
@@ -15478,9 +15478,9 @@ There is no extra whitespace here { false, false, false, false, false } not contains only true - +
- +
@@ -15492,9 +15492,9 @@ There is no extra whitespace here { true, true, true, true, true } contains only true - +
- +
@@ -15506,9 +15506,9 @@ There is no extra whitespace here { true, true, false, true, true } not contains only true - +
- +
@@ -15520,9 +15520,9 @@ There is no extra whitespace here { false, false, false, false, false } not contains only true - +
- +
@@ -15574,9 +15574,9 @@ There is no extra whitespace here true - +
- +
@@ -15628,11 +15628,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -15652,7 +15652,7 @@ There is no extra whitespace here { { 0, 1, 2, 3, 5 }, { 4, -3, -2, 5, 0 }, { 0, 0, 0, 5, 0 }, { 0, -5, 0, 5, 0 }, { 1, 0, 0, -1, 5 } } not any match ( contains element 0 and contains element 10 ) - +
@@ -15663,7 +15663,7 @@ There is no extra whitespace here { 1, 2, 3, 4, 5 } any match matches undescribed predicate - +
@@ -15715,9 +15715,9 @@ There is no extra whitespace here true - +
- +
@@ -15769,11 +15769,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -15786,9 +15786,9 @@ There is no extra whitespace here { true, true, true, true, true } contains at least one true - +
- +
@@ -15800,9 +15800,9 @@ There is no extra whitespace here { } not contains at least one true - +
- +
@@ -15814,9 +15814,9 @@ There is no extra whitespace here { false, false, true, false, false } contains at least one true - +
- +
@@ -15828,9 +15828,9 @@ There is no extra whitespace here { false, false, false, false, false } not contains at least one true - +
- +
@@ -15842,9 +15842,9 @@ There is no extra whitespace here { true, true, true, true, true } contains at least one true - +
- +
@@ -15856,9 +15856,9 @@ There is no extra whitespace here { false, false, true, false, false } contains at least one true - +
- +
@@ -15870,9 +15870,9 @@ There is no extra whitespace here { false, false, false, false, false } not contains at least one true - +
- +
@@ -15924,9 +15924,9 @@ There is no extra whitespace here true - +
- +
@@ -15978,11 +15978,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -16002,7 +16002,7 @@ There is no extra whitespace here { { 0, 1, 2, 3, 5 }, { 4, -3, -2, 5, 0 }, { 0, 0, 0, 5, 0 }, { 0, -5, 0, 5, 0 }, { 1, 0, 0, -1, 5 } } not none match ( contains element 0 and contains element 1 ) - +
@@ -16013,7 +16013,7 @@ There is no extra whitespace here { 1, 2, 3, 4, 5 } none match matches undescribed predicate - +
@@ -16065,9 +16065,9 @@ There is no extra whitespace here true - +
- +
@@ -16119,11 +16119,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -16136,9 +16136,9 @@ There is no extra whitespace here { true, true, true, true, true } not contains no true - +
- +
@@ -16150,9 +16150,9 @@ There is no extra whitespace here { } contains no true - +
- +
@@ -16164,9 +16164,9 @@ There is no extra whitespace here { false, false, true, false, false } not contains no true - +
- +
@@ -16178,9 +16178,9 @@ There is no extra whitespace here { false, false, false, false, false } contains no true - +
- +
@@ -16192,9 +16192,9 @@ There is no extra whitespace here { true, true, true, true, true } not contains no true - +
- +
@@ -16206,9 +16206,9 @@ There is no extra whitespace here { false, false, true, false, false } not contains no true - +
- +
@@ -16220,9 +16220,9 @@ There is no extra whitespace here { false, false, false, false, false } contains no true - +
- +
@@ -16274,9 +16274,9 @@ There is no extra whitespace here true - +
- +
@@ -16328,11 +16328,11 @@ There is no extra whitespace here !false - +
- +
- +
@@ -16392,7 +16392,7 @@ There is no extra whitespace here { {?}, {?}, {?} } has size == 3 - +
@@ -16403,7 +16403,7 @@ There is no extra whitespace here {?} has size == 12 - +
@@ -16414,9 +16414,9 @@ There is no extra whitespace here {?} has size == 13 - +
- + @@ -16483,13 +16483,13 @@ There is no extra whitespace here Approx( 1.23 ) != 1.25 - +
- +
- +
@@ -16501,7 +16501,7 @@ There is no extra whitespace here { } is approx: { } - +
@@ -16521,9 +16521,9 @@ There is no extra whitespace here { 1.0, 2.0, 3.0 } is approx: { 1.0, 2.0, 3.0 } - +
- +
@@ -16535,9 +16535,9 @@ There is no extra whitespace here { 1.0, 2.0, 3.0 } not is approx: { 1.0, 2.0, 3.0, 4.0 } - +
- +
@@ -16573,11 +16573,11 @@ There is no extra whitespace here { 1.0, 2.0, 3.0 } is approx: { 1.5, 2.5, 3.5 } - +
- +
- +
@@ -16589,7 +16589,7 @@ There is no extra whitespace here { } is approx: { 1.0, 2.0 } - +
@@ -16600,9 +16600,9 @@ There is no extra whitespace here { 2.0, 4.0, 6.0 } is approx: { 1.0, 3.0, 5.0 } - +
- +
@@ -16630,7 +16630,7 @@ There is no extra whitespace here { 1, 2, 3 } Contains: 2 - +
@@ -16697,7 +16697,7 @@ There is no extra whitespace here { 1, 2, 3 } Contains: { 1, 2 } - +
@@ -16708,7 +16708,7 @@ There is no extra whitespace here { 1, 2, 3 } ( Contains: 1 and Contains: 2 ) - +
@@ -16759,7 +16759,7 @@ There is no extra whitespace here { 1, 2, 3 } Equals: { 1, 2, 3 } - +
@@ -16818,9 +16818,9 @@ There is no extra whitespace here { 1, 3, 2 } UnorderedEquals: { 1, 2, 3 } - +
- +
@@ -16840,7 +16840,7 @@ There is no extra whitespace here { } Contains: 1 - +
@@ -16859,7 +16859,7 @@ There is no extra whitespace here { 1, 2, 3 } Contains: { 1, 2, 4 } - +
@@ -16894,7 +16894,7 @@ There is no extra whitespace here { 1, 2, 3 } Equals: { } - +
@@ -16929,9 +16929,9 @@ There is no extra whitespace here { 3, 1 } UnorderedEquals: { 1, 2, 3 } - +
- + @@ -16958,13 +16958,13 @@ There is no extra whitespace here thisThrows() - + unexpected exception - + @@ -16978,7 +16978,7 @@ There is no extra whitespace here expected exception - + @@ -16992,7 +16992,7 @@ There is no extra whitespace here expected exception - + @@ -17006,31 +17006,31 @@ There is no extra whitespace here expected exception - +
unexpected exception - +
- + - + - + - + - + - +
@@ -17042,7 +17042,7 @@ There is no extra whitespace here "normal string" == "normal string" - +
@@ -17053,7 +17053,7 @@ There is no extra whitespace here "" == "" - +
@@ -17064,7 +17064,7 @@ There is no extra whitespace here "smith &amp; jones" == "smith &amp; jones" - +
@@ -17075,7 +17075,7 @@ There is no extra whitespace here "smith &lt; jones" == "smith &lt; jones" - +
@@ -17096,7 +17096,7 @@ There is no extra whitespace here "smith ]]&gt; jones" - +
@@ -17119,7 +17119,7 @@ There is no extra whitespace here "don't &quot;quote&quot; me on that" - +
@@ -17130,7 +17130,7 @@ There is no extra whitespace here "[\x01]" == "[\x01]" - +
@@ -17141,9 +17141,9 @@ There is no extra whitespace here "[\x7F]" == "[\x7F]" - +
- + @@ -17156,10 +17156,11 @@ There is no extra whitespace here " ( contains: "attr1="true"" and contains: "attr2="false"" ) - + - + + @@ -17266,7 +17267,7 @@ There is no extra whitespace here 0.0 == 0 - + @@ -17293,7 +17294,7 @@ There is no extra whitespace here "{ 42, 250 }" == "{ 42, 250 }" - +
@@ -17337,7 +17338,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17380,9 +17381,9 @@ There is no extra whitespace here 1 == 1 - +
- + @@ -17393,7 +17394,7 @@ There is no extra whitespace here 0x != 0 - + @@ -17412,7 +17413,7 @@ There is no extra whitespace here true - + @@ -17431,7 +17432,7 @@ There is no extra whitespace here false - + @@ -17450,7 +17451,7 @@ There is no extra whitespace here true - + @@ -17469,7 +17470,7 @@ There is no extra whitespace here false - +
@@ -17521,7 +17522,7 @@ There is no extra whitespace here 0 == 0 - +
@@ -17572,7 +17573,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17623,7 +17624,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17674,7 +17675,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17725,7 +17726,7 @@ There is no extra whitespace here 1 == 1 - +
@@ -17776,9 +17777,9 @@ There is no extra whitespace here 2 == 2 - +
- + @@ -17813,7 +17814,7 @@ There is no extra whitespace here 1 == 1 - + @@ -17848,7 +17849,7 @@ There is no extra whitespace here 1 == 1 - + @@ -17903,10 +17904,16 @@ There is no extra whitespace here 1 == 1 - + - + + skipping because answer = 41 + + + skipping because answer = 43 + + @@ -17917,7 +17924,7 @@ There is no extra whitespace here Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo) - + @@ -17944,7 +17951,7 @@ There is no extra whitespace here 1.3859038243 == Approx( 1.3859038243 ) - + @@ -17963,25 +17970,25 @@ There is no extra whitespace here 0 == 0 - +
- +
- +
- +
- +
- +
- + @@ -17992,49 +17999,53 @@ There is no extra whitespace here 3 == 4 - + + + - + +
- + +
- +
- + - + - + - + Previous info should not be seen - + previous unscoped info SHOULD not be seen - + - + - + @@ -18047,7 +18058,7 @@ There is no extra whitespace here 9223372036854775807 (0x) - +
@@ -18059,7 +18070,7 @@ There is no extra whitespace here 0 > 1 - +
@@ -18070,7 +18081,7 @@ There is no extra whitespace here 1 > 1 - +
@@ -18081,7 +18092,7 @@ There is no extra whitespace here 2 > 1 - +
@@ -18092,7 +18103,7 @@ There is no extra whitespace here 3 > 1 - +
@@ -18103,7 +18114,7 @@ There is no extra whitespace here 4 > 1 - +
@@ -18114,7 +18125,7 @@ There is no extra whitespace here 5 > 1 - +
@@ -18125,7 +18136,7 @@ There is no extra whitespace here 6 > 1 - +
@@ -18136,7 +18147,7 @@ There is no extra whitespace here 7 > 1 - +
@@ -18147,7 +18158,7 @@ There is no extra whitespace here 8 > 1 - +
@@ -18158,9 +18169,9 @@ There is no extra whitespace here 9 > 1 - +
- + @@ -18251,7 +18262,7 @@ There is no extra whitespace here 1 == 0 - + @@ -18262,7 +18273,7 @@ There is no extra whitespace here Catch::makeStream( "%debug" ) - +
@@ -18274,7 +18285,7 @@ There is no extra whitespace here !false - +
@@ -18285,7 +18296,7 @@ There is no extra whitespace here true - +
@@ -18296,9 +18307,9 @@ There is no extra whitespace here {?} == {?} - +
- + @@ -18309,7 +18320,7 @@ There is no extra whitespace here 19.0 == 19.0 - + @@ -18376,7 +18387,7 @@ There is no extra whitespace here 1 == 1 - + @@ -18397,7 +18408,7 @@ There is no extra whitespace here they are not cleared after warnings - +
@@ -18410,9 +18421,9 @@ There is no extra whitespace here 1 == 2 - +
- +
@@ -18424,9 +18435,9 @@ There is no extra whitespace here 1 != 2 - +
- +
@@ -18438,11 +18449,11 @@ There is no extra whitespace here 1 < 2 - +
- +
- +
@@ -18471,32 +18482,33 @@ There is no extra whitespace here 1 != 2 - +
- + - +
- +
- +
- +
- + +
- +
- +
- + a! b1! @@ -18513,7 +18525,7 @@ b1! "7" == "7" - + @@ -18524,7 +18536,7 @@ b1! {?} == {?} - + @@ -18567,7 +18579,7 @@ b1! 0.088096521 == Approx( 0.088096521 ) - + @@ -18594,10 +18606,10 @@ b1! -1.9599639845 == Approx( -1.9599639845 ) - + - + @@ -18633,7 +18645,7 @@ b1! false - + @@ -18652,7 +18664,7 @@ b1! {null string} == {null string} - + @@ -18663,7 +18675,7 @@ b1! 0 == 0 - + @@ -18676,7 +18688,7 @@ b1! "{ { 42, "Arthur" }, { "Ford", 24 } }" - +
@@ -18688,7 +18700,7 @@ b1! { } Equals: { } - +
@@ -18715,7 +18727,7 @@ b1! { Value1 } Equals: { Value1 } - +
@@ -18742,9 +18754,9 @@ b1! { Value1, Value2, Value3 } Equals: { Value1, Value2, Value3 } - +
- + @@ -18755,7 +18767,7 @@ b1! 0 == 0 - + @@ -18769,7 +18781,7 @@ b1! true - + @@ -18786,7 +18798,7 @@ b1! false - + @@ -18827,7 +18839,7 @@ b1! true - +
@@ -18847,7 +18859,7 @@ b1! 2 != 1 - +
@@ -18858,9 +18870,9 @@ b1! 1 != 2 - +
- +
@@ -18880,7 +18892,7 @@ b1! "azcdefcg" == "azcdefcg" - +
@@ -18899,7 +18911,7 @@ b1! "abzdefzg" == "abzdefzg" - +
@@ -18918,7 +18930,7 @@ b1! "zbcdefcg" == "zbcdefcg" - +
@@ -18937,7 +18949,7 @@ b1! "abcdefcz" == "abcdefcz" - +
@@ -18956,7 +18968,7 @@ b1! "replaced" == "replaced" - +
@@ -18975,7 +18987,7 @@ b1! "abcdefcg" == "abcdefcg" - +
@@ -18994,9 +19006,9 @@ b1! "didn|'t" == "didn|'t" - +
- + @@ -19007,7 +19019,7 @@ b1! Catch::makeStream( "%somestream" ) - + @@ -19090,7 +19102,7 @@ b1! 1000.0 == 1000 (0x) - + @@ -19181,7 +19193,7 @@ b1! 128 >= 100 - + @@ -19272,22 +19284,23 @@ b1! 128 >= 100 - + - +
- +
- + +
- +
- + @@ -19301,7 +19314,7 @@ b1! false - + @@ -19318,7 +19331,7 @@ b1! false - + @@ -19329,10 +19342,13 @@ b1! { {?}, {?} } ( Contains: {?} and Contains: {?} ) - + - + + skipping because answer = 43 + + @@ -19359,7 +19375,7 @@ b1! { abc, def } Equals: { abc, def } - + @@ -19402,7 +19418,7 @@ b1! false - + @@ -19429,7 +19445,7 @@ b1! true - +
@@ -19441,7 +19457,7 @@ b1! "{ }" == "{ }" - +
@@ -19452,7 +19468,7 @@ b1! "{ { "one", 1 } }" == "{ { "one", 1 } }" - +
@@ -19465,9 +19481,9 @@ b1! "{ { "abc", 1 }, { "def", 2 }, { "ghi", 3 } }" - +
- + @@ -19478,7 +19494,7 @@ b1! "{ 34, "xyzzy" }" == "{ 34, "xyzzy" }" - + @@ -19489,7 +19505,7 @@ b1! "{ 34, "xyzzy" }" == "{ 34, "xyzzy" }" - +
@@ -19501,7 +19517,7 @@ b1! "{ }" == "{ }" - +
@@ -19512,7 +19528,7 @@ b1! "{ "one" }" == "{ "one" }" - +
@@ -19525,9 +19541,9 @@ b1! "{ "abc", "def", "ghi" }" - +
- + @@ -19540,7 +19556,7 @@ b1! "{ { "green", 55 } }" - + @@ -19559,7 +19575,7 @@ b1! true - + @@ -19598,7 +19614,7 @@ b1! "{?}" == "{?}" - + @@ -19611,7 +19627,7 @@ b1! "StringMaker<has_maker>" - + @@ -19624,7 +19640,7 @@ b1! "StringMaker<has_maker_and_operator>" - + @@ -19635,7 +19651,7 @@ b1! "{?}" == "{?}" - + @@ -19648,7 +19664,7 @@ b1! "operator<<( has_operator )" - + @@ -19661,7 +19677,7 @@ b1! "operator<<( has_template_operator )" - + @@ -19674,7 +19690,7 @@ b1! "{ StringMaker<has_maker> }" - + @@ -19687,7 +19703,7 @@ b1! "{ StringMaker<has_maker_and_operator> }" - + @@ -19700,7 +19716,7 @@ b1! "{ operator<<( has_operator ) }" - + @@ -19735,7 +19751,7 @@ b1! 4 == 4 - + @@ -19770,7 +19786,7 @@ b1! 6 == 6 - + @@ -19789,16 +19805,17 @@ b1! magic.tag == magic.tag - + - + + Why would you throw a std::string? - + @@ -19809,7 +19826,7 @@ b1! ""wide load"" == ""wide load"" - + @@ -19820,7 +19837,7 @@ b1! ""wide load"" == ""wide load"" - + @@ -19831,7 +19848,7 @@ b1! ""wide load"" == ""wide load"" - + @@ -19842,7 +19859,7 @@ b1! ""wide load"" == ""wide load"" - + @@ -19871,7 +19888,7 @@ b1! "Unknown enum value 10" - + @@ -19890,7 +19907,7 @@ b1! "1" == "1" - + @@ -19909,7 +19926,7 @@ b1! "E2{1}" == "E2{1}" - + @@ -19928,7 +19945,7 @@ b1! "1" == "1" - + @@ -19947,7 +19964,7 @@ b1! "{ }" == "{ }" - + @@ -19966,7 +19983,7 @@ b1! "{ 1.2f, 0 }" == "{ 1.2f, 0 }" - + @@ -19977,7 +19994,7 @@ b1! "{ 0 }" == "{ 0 }" - + @@ -19990,7 +20007,7 @@ b1! "{ "hello", "world" }" - + @@ -20003,7 +20020,7 @@ b1! "{ { 42 }, { }, 1.2f }" - + @@ -20038,7 +20055,7 @@ b1! 0.95 == 0.95 - +
@@ -20058,7 +20075,7 @@ b1! 0 == 0 - +
@@ -20102,9 +20119,9 @@ b1! 0 == 0 - +
- +
@@ -20156,9 +20173,9 @@ b1! 2 == 2 - +
- +
@@ -20177,7 +20194,7 @@ b1! 0 == 0 - +
@@ -20204,7 +20221,7 @@ b1! 1 == 1 - +
@@ -20231,7 +20248,7 @@ b1! 2 == 2 - +
@@ -20250,9 +20267,9 @@ b1! 1 == 1 - +
- + @@ -20273,7 +20290,7 @@ b1! "{ { "hello" }, { "world" } }" - + @@ -20300,7 +20317,7 @@ b1! "{ true, false }" == "{ true, false }" - + @@ -20327,7 +20344,7 @@ b1! "{ 42, 250 }" == "{ 42, 250 }" - + @@ -20354,7 +20371,7 @@ b1! "{ 42, 250 }" == "{ 42, 250 }" - + @@ -20383,7 +20400,7 @@ b1! "{ "hello", "world" }" - + @@ -20419,7 +20436,7 @@ b1! 10 >= 10 - + @@ -20463,9 +20480,9 @@ b1! 0 == 0 - + - + @@ -20500,7 +20517,7 @@ b1! 10 >= 10 - + @@ -20535,9 +20552,9 @@ b1! 5 >= 5 - + - + @@ -20556,7 +20573,7 @@ b1! 310016000 ns > 100 ms - + @@ -20583,17 +20600,17 @@ b1! 23.0 == 23.0 - +
- +
- +
- + - - + + From 6f2ebd340e2a6ca286910a7ae55d983f8a117905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Thu, 22 Dec 2022 10:25:40 +0100 Subject: [PATCH 6/9] Fix license header --- tests/ExtraTests/X93-AllSkipped.cpp | 2 +- tests/SelfTest/UsageTests/Skip.tests.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ExtraTests/X93-AllSkipped.cpp b/tests/ExtraTests/X93-AllSkipped.cpp index f298116974..8e7d0afe66 100644 --- a/tests/ExtraTests/X93-AllSkipped.cpp +++ b/tests/ExtraTests/X93-AllSkipped.cpp @@ -1,7 +1,7 @@ // Copyright Catch2 Authors // Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at +// (See accompanying file LICENSE.txt or copy at // https://www.boost.org/LICENSE_1_0.txt) // SPDX-License-Identifier: BSL-1.0 diff --git a/tests/SelfTest/UsageTests/Skip.tests.cpp b/tests/SelfTest/UsageTests/Skip.tests.cpp index 70feafaf44..6bd4189b43 100644 --- a/tests/SelfTest/UsageTests/Skip.tests.cpp +++ b/tests/SelfTest/UsageTests/Skip.tests.cpp @@ -1,7 +1,7 @@ // Copyright Catch2 Authors // Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at +// (See accompanying file LICENSE.txt or copy at // https://www.boost.org/LICENSE_1_0.txt) // SPDX-License-Identifier: BSL-1.0 From 54cc2fa929a53ae67b77e0bda189476e3789663b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Thu, 22 Dec 2022 10:28:43 +0100 Subject: [PATCH 7/9] Cleanup xml reporter handling of tag ending --- src/catch2/reporters/catch_reporter_xml.cpp | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/catch2/reporters/catch_reporter_xml.cpp b/src/catch2/reporters/catch_reporter_xml.cpp index 03ffff2c22..011f90067c 100644 --- a/src/catch2/reporters/catch_reporter_xml.cpp +++ b/src/catch2/reporters/catch_reporter_xml.cpp @@ -170,16 +170,18 @@ namespace Catch { void XmlReporter::sectionEnded( SectionStats const& sectionStats ) { StreamingReporterBase::sectionEnded( sectionStats ); - if( --m_sectionDepth > 0 ) { - XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); - e.writeAttribute( "successes"_sr, sectionStats.assertions.passed ); - e.writeAttribute( "failures"_sr, sectionStats.assertions.failed ); - e.writeAttribute( "expectedFailures"_sr, sectionStats.assertions.failedButOk ); - e.writeAttribute( "skipped"_sr, sectionStats.assertions.skipped > 0 ); - - if ( m_config->showDurations() == ShowDurations::Always ) - e.writeAttribute( "durationInSeconds"_sr, sectionStats.durationInSeconds ); - + if ( --m_sectionDepth > 0 ) { + { + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); + e.writeAttribute( "successes"_sr, sectionStats.assertions.passed ); + e.writeAttribute( "failures"_sr, sectionStats.assertions.failed ); + e.writeAttribute( "expectedFailures"_sr, sectionStats.assertions.failedButOk ); + e.writeAttribute( "skipped"_sr, sectionStats.assertions.skipped > 0 ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds"_sr, sectionStats.durationInSeconds ); + } + // Ends assertion tag m_xml.endElement(); } } From e08e62d062b1712897b97b9bf22a06403720ade9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Thu, 5 Jan 2023 23:40:19 +0100 Subject: [PATCH 8/9] --allow-running-no-tests also allows all tests to be skipped --- src/catch2/catch_session.cpp | 3 ++- tests/CMakeLists.txt | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp index c89368815f..43465f0c46 100644 --- a/src/catch2/catch_session.cpp +++ b/src/catch2/catch_session.cpp @@ -342,7 +342,8 @@ namespace Catch { } if ( totals.testCases.total() > 0 && - totals.testCases.total() == totals.testCases.skipped ) { + totals.testCases.total() == totals.testCases.skipped + && !m_config->zeroTestsCountAsSuccess() ) { return 4; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 464fb683ac..023f478b83 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -273,6 +273,10 @@ add_test(NAME TestSpecs::OverrideFailureWithNoMatchedTests COMMAND $ "___nonexistent_test___" --allow-running-no-tests ) +add_test(NAME TestSpecs::OverrideAllSkipFailure + COMMAND $ "tests can be skipped dynamically at runtime" --allow-running-no-tests +) + add_test(NAME TestSpecs::NonMatchingTestSpecIsRoundTrippable COMMAND $ Tracker, "this test does not exist" "[nor does this tag]" ) From 6309b3cd903b6feb88563a9a120e29777e4cd334 Mon Sep 17 00:00:00 2001 From: Philip Salzmann Date: Mon, 9 Jan 2023 15:48:19 +0100 Subject: [PATCH 9/9] Add docs for SKIP macro, deprecate IEventListener::skipTest --- docs/Readme.md | 1 + docs/command-line.md | 8 +-- docs/deprecations.md | 9 +++ docs/skipping.md | 72 +++++++++++++++++++ .../interfaces/catch_interfaces_reporter.hpp | 7 +- 5 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 docs/skipping.md diff --git a/docs/Readme.md b/docs/Readme.md index 52eb64a50f..126ee1be9b 100644 --- a/docs/Readme.md +++ b/docs/Readme.md @@ -11,6 +11,7 @@ Once you're up and running consider the following reference material. * [Logging macros](logging.md#top) * [Test cases and sections](test-cases-and-sections.md#top) * [Test fixtures](test-fixtures.md#top) +* [Skipping tests at runtime](skipping.md#top) * [Reporters (output customization)](reporters.md#top) * [Event Listeners](event-listeners.md#top) * [Data Generators (value parameterized tests)](generators.md#top) diff --git a/docs/command-line.md b/docs/command-line.md index 6fc99f59e7..e7a448b165 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -561,10 +561,10 @@ processes, as is done with the [Bazel test sharding](https://docs.bazel.build/ve > Introduced in Catch2 3.0.1. -By default, Catch2 test binaries return non-0 exit code if no tests were -run, e.g. if the binary was compiled with no tests, or the provided test -spec matched no tests. This flag overrides that, so a test run with no -tests still returns 0. +By default, Catch2 test binaries return non-0 exit code if no tests were run, +e.g. if the binary was compiled with no tests, the provided test spec matched no +tests, or all tests [were skipped at runtime](skipping.md#top). This flag +overrides that, so a test run with no tests still returns 0. ## Output verbosity ``` diff --git a/docs/deprecations.md b/docs/deprecations.md index 2c9bf5517b..5926f2c541 100644 --- a/docs/deprecations.md +++ b/docs/deprecations.md @@ -26,6 +26,15 @@ to accurately probe the environment for this information so the flag where it will export `BAZEL_TEST=1` for purposes like the above. Catch2 will now instead inspect the environment instead of relying on build configuration. +### `IEventLister::skipTest( TestCaseInfo const& testInfo )` + +This event (including implementations in derived classes such as `ReporterBase`) +is deprecated and will be removed in the next major release. It is currently +invoked for all test cases that are not going to be executed due to the test run +being aborted (when using `--abort` or `--abortx`). It is however +**NOT** invoked for test cases that are [explicitly skipped using the `SKIP` +macro](skipping.md#top). + --- [Home](Readme.md#top) diff --git a/docs/skipping.md b/docs/skipping.md new file mode 100644 index 0000000000..edd2a1a1ae --- /dev/null +++ b/docs/skipping.md @@ -0,0 +1,72 @@ + +# Skipping Test Cases at Runtime + +> [Introduced](https://github.com/catchorg/Catch2/pull/2360) in Catch2 X.Y.Z. + +In some situations it may not be possible to meaningfully execute a test case, for example when the system under test is missing certain hardware capabilities. +If the required conditions can only be determined at runtime, it often doesn't make sense to consider such a test case as either passed or failed, because it simply can not run at all. +To properly express such scenarios, Catch2 allows to explicitly _skip_ test cases, using the `SKIP` macro: + +**SKIP(** _message expression_ **)** + +Example usage: + +```c++ +TEST_CASE("copy files between drives") { + if(getNumberOfHardDrives() < 2) { + SKIP("at least two hard drives required"); + } + // ... +} +``` + +This test case is then reported as _skipped_ instead of _passed_ or _failed_. + +The `SKIP` macro behaves similarly to an explicit [`FAIL`](logging.md#top), in that it is the last expression that will be executed: + +```c++ +TEST_CASE("my test") { + printf("foo"); + SKIP(); + printf("bar"); // not printed +} +``` + +However a failed assertion _before_ a `SKIP` still causes the entire test case to fail: + +```c++ +TEST_CASE("failing test") { + CHECK(1 == 2); + SKIP(); +} +``` + +## Interaction with Sections and Generators + +Sections, nested sections as well as test cases with [generators](generators.md#top) can all be individually skipped, with the rest executing as usual: + +```c++ +TEST_CASE("complex test case") { + int value = GENERATE(2, 4, 6); + SECTION("a") { + SECTION("a1") { CHECK(value < 8); } + SECTION("a2") { + if (value == 4) { + SKIP(); + } + CHECK(value % 2 == 0); + } + } +} +``` + +This test case will report 5 passing assertions; one for each of the three values in section `a1`, as well as one for each in `a2`, except for when `value == 4`. + +Note that as soon as one section is skipped, the entire test case will be reported as _skipped_ (unless there is a failing assertion, in which case it will be reported as _failed_ instead). + +If all test cases in a run are skipped, Catch2 returns a non-zero exit code by default. +This can be overridden using the [--allow-running-no-tests](command-line.md#no-tests-override) flag. + +--- + +[Home](Readme.md#top) diff --git a/src/catch2/interfaces/catch_interfaces_reporter.hpp b/src/catch2/interfaces/catch_interfaces_reporter.hpp index 5f28636332..da0112e3f8 100644 --- a/src/catch2/interfaces/catch_interfaces_reporter.hpp +++ b/src/catch2/interfaces/catch_interfaces_reporter.hpp @@ -242,7 +242,12 @@ namespace Catch { */ virtual void testRunEnded( TestRunStats const& testRunStats ) = 0; - //! Called with test cases that are skipped due to the test run aborting + /** + * Called with test cases that are skipped due to the test run aborting. + * NOT called for test cases that are explicitly skipped using the `SKIP` macro. + * + * Deprecated - will be removed in the next major release. + */ virtual void skipTest( TestCaseInfo const& testInfo ) = 0; //! Called if a fatal error (signal/structured exception) occured