Add new SKIP macro for skipping tests at runtime

This adds a new `SKIP` macro for dynamically skipping tests at runtime.
The "skipped" status of a test case is treated as a first-class citizen,
like "succeeded" or "failed", and is reported with a new color on the
console.

src/catch2/catch_test_macros.hpp

@@ -49,6 +49,7 @@
   #define CATCH_FAIL( ... ) INTERNAL_CATCH_MSG( "CATCH_FAIL", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, __VA_ARGS__ )
   #define CATCH_FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( "CATCH_FAIL_CHECK", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ )
   #define CATCH_SUCCEED( ... ) INTERNAL_CATCH_MSG( "CATCH_SUCCEED", Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ )
+  #define CATCH_SKIP( ... ) INTERNAL_CATCH_MSG( "SKIP", Catch::ResultWas::ExplicitSkip, Catch::ResultDisposition::Normal, __VA_ARGS__ )
 
 
   #if !defined(CATCH_CONFIG_RUNTIME_STATIC_REQUIRE)
@@ -102,6 +103,7 @@
   #define CATCH_FAIL( ... ) (void)(0)
   #define CATCH_FAIL_CHECK( ... ) (void)(0)
   #define CATCH_SUCCEED( ... ) (void)(0)
+  #define CATCH_SKIP( ... ) (void)(0)
 
   #define CATCH_STATIC_REQUIRE( ... )       (void)(0)
   #define CATCH_STATIC_REQUIRE_FALSE( ... ) (void)(0)
@@ -146,6 +148,7 @@
   #define FAIL( ... ) INTERNAL_CATCH_MSG( "FAIL", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, __VA_ARGS__ )
   #define FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( "FAIL_CHECK", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ )
   #define SUCCEED( ... ) INTERNAL_CATCH_MSG( "SUCCEED", Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ )
+  #define SKIP( ... ) INTERNAL_CATCH_MSG( "SKIP", Catch::ResultWas::ExplicitSkip, Catch::ResultDisposition::Normal, __VA_ARGS__ )
 
 
   #if !defined(CATCH_CONFIG_RUNTIME_STATIC_REQUIRE)
@@ -198,6 +201,7 @@
   #define FAIL( ... ) (void)(0)
   #define FAIL_CHECK( ... ) (void)(0)
   #define SUCCEED( ... ) (void)(0)
+  #define SKIP( ... ) (void)(0)
 
   #define STATIC_REQUIRE( ... )       (void)(0)
   #define STATIC_REQUIRE_FALSE( ... ) (void)(0)

src/catch2/catch_totals.cpp

@@ -14,21 +14,23 @@ namespace Catch {
         diff.passed = passed - other.passed;
         diff.failed = failed - other.failed;
         diff.failedButOk = failedButOk - other.failedButOk;
+        diff.skipped = skipped - other.skipped;
         return diff;
     }
 
     Counts& Counts::operator += ( Counts const& other ) {
         passed += other.passed;
         failed += other.failed;
         failedButOk += other.failedButOk;
+        skipped += other.skipped;
         return *this;
     }
 
     std::uint64_t Counts::total() const {
-        return passed + failed + failedButOk;
+        return passed + failed + failedButOk + skipped;
     }
     bool Counts::allPassed() const {
-        return failed == 0 && failedButOk == 0;
+        return failed == 0 && failedButOk == 0 && skipped == 0;
     }
     bool Counts::allOk() const {
         return failed == 0;
@@ -53,6 +55,8 @@ namespace Catch {
             ++diff.testCases.failed;
         else if( diff.assertions.failedButOk > 0 )
             ++diff.testCases.failedButOk;
+        else if ( diff.assertions.skipped > 0 )
+            ++ diff.testCases.skipped;
         else
             ++diff.testCases.passed;
         return diff;

src/catch2/catch_totals.hpp

@@ -24,6 +24,7 @@ namespace Catch {
         std::uint64_t passed = 0;
         std::uint64_t failed = 0;
         std::uint64_t failedButOk = 0;
+        std::uint64_t skipped = 0;
     };
 
     struct Totals {

src/catch2/internal/catch_assertion_handler.cpp

@@ -52,6 +52,13 @@ namespace Catch {
             throw Catch::TestFailureException();
 #else
             CATCH_ERROR( "Test failure requires aborting test!" );
+#endif
+        }
+        if ( m_reaction.shouldSkip ) {
+#if !defined( CATCH_CONFIG_DISABLE_EXCEPTIONS )
+            throw Catch::TestSkipException();
+#else
+            CATCH_ERROR( "Explicitly skipping tests during runtime requires exceptions" );
 #endif
         }
     }

src/catch2/internal/catch_assertion_handler.hpp

@@ -22,6 +22,7 @@ namespace Catch {
     struct AssertionReaction {
         bool shouldDebugBreak = false;
         bool shouldThrow = false;
+        bool shouldSkip = false;
     };
 
     class AssertionHandler {

src/catch2/internal/catch_console_colour.hpp

@@ -41,6 +41,7 @@ namespace Catch {
 
             Error = BrightRed,
             Success = Green,
+            Skip = Blue,
 
             OriginalExpression = Cyan,
             ReconstructedExpression = BrightYellow,

src/catch2/internal/catch_exception_translator_registry.cpp

@@ -44,6 +44,9 @@ namespace Catch {
         catch( TestFailureException& ) {
             std::rethrow_exception(std::current_exception());
         }
+        catch( TestSkipException& ) {
+            std::rethrow_exception(std::current_exception());
+        }
         catch( std::exception const& ex ) {
             return ex.what();
         }

src/catch2/internal/catch_result_type.hpp

@@ -16,6 +16,8 @@ namespace Catch {
         Ok = 0,
         Info = 1,
         Warning = 2,
+        // TODO: Should explicit skip be considered "not OK" (cf. isOk)? I.e., should it have the failure bit?
+        ExplicitSkip = 4,
 
         FailureBit = 0x10,
 

src/catch2/internal/catch_run_context.cpp

@@ -236,6 +236,9 @@ namespace Catch {
         if (result.getResultType() == ResultWas::Ok) {
             m_totals.assertions.passed++;
             m_lastAssertionPassed = true;
+        } else if (result.getResultType() == ResultWas::ExplicitSkip) {
+            m_totals.assertions.skipped++;
+            m_lastAssertionPassed = true;
         } else if (!result.succeeded()) {
             m_lastAssertionPassed = false;
             if (result.isOk()) {
@@ -443,6 +446,8 @@ namespace Catch {
             duration = timer.getElapsedSeconds();
         } CATCH_CATCH_ANON (TestFailureException&) {
             // This just means the test was aborted due to failure
+        } CATCH_CATCH_ANON (TestSkipException&) {
+            // This just means the test was explicitly skipped
         } CATCH_CATCH_ALL {
             // Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions
             // are reported without translation at the point of origin.
@@ -534,8 +539,13 @@ namespace Catch {
         data.message = static_cast<std::string>(message);
         AssertionResult assertionResult{ m_lastAssertionInfo, data };
         assertionEnded( assertionResult );
-        if( !assertionResult.isOk() )
+        if ( !assertionResult.isOk() ) {
             populateReaction( reaction );
+        } else if ( resultType == ResultWas::ExplicitSkip ) {
+            // TODO: Need to handle this explicitly, as ExplicitSkip is
+            // considered "OK"
+            reaction.shouldSkip = true;
+        }
     }
     void RunContext::handleUnexpectedExceptionNotThrown(
             AssertionInfo const& info,

src/catch2/internal/catch_test_failure_exception.hpp

@@ -13,6 +13,9 @@ namespace Catch {
     //! Used to signal that an assertion macro failed
     struct TestFailureException{};
 
+    //! Used to signal that the remainder of a test should be skipped
+    struct TestSkipException{};
+
 } // namespace Catch
 
 #endif // CATCH_TEST_FAILURE_EXCEPTION_HPP_INCLUDED

src/catch2/reporters/catch_reporter_compact.cpp

@@ -159,6 +159,7 @@ class AssertionPrinter {
         case ResultWas::Unknown:
         case ResultWas::FailureBit:
         case ResultWas::Exception:
+        case ResultWas::ExplicitSkip:
             printResultType(Colour::Error, "** internal error **");
             break;
         }

src/catch2/reporters/catch_reporter_console.cpp

@@ -109,6 +109,14 @@ class ConsoleAssertionPrinter {
             if (_stats.infoMessages.size() > 1)
                 messageLabel = "explicitly with messages";
             break;
+        case ResultWas::ExplicitSkip:
+            colour = Colour::Skip;
+            passOrFail = "SKIPPED"_sr;
+            if (_stats.infoMessages.size() == 1)
+                messageLabel = "explicitly with message";
+            if (_stats.infoMessages.size() > 1)
+                messageLabel = "explicitly with messages";
+            break;
             // These cases are here to prevent compiler warnings
         case ResultWas::Unknown:
         case ResultWas::FailureBit:
@@ -183,13 +191,16 @@ std::size_t makeRatio( std::uint64_t number, std::uint64_t total ) {
     return (ratio == 0 && number > 0) ? 1 : static_cast<std::size_t>(ratio);
 }
 
-std::size_t& findMax( std::size_t& i, std::size_t& j, std::size_t& k ) {
-    if (i > j && i > k)
+std::size_t&
+findMax( std::size_t& i, std::size_t& j, std::size_t& k, std::size_t& l ) {
+    if (i > j && i > k && i > l)
         return i;
-    else if (j > k)
+    else if (j > k && j > l)
         return j;
-    else
+    else if (k > l)
         return k;
+    else
+        return l;
 }
 
 enum class Justification { Left, Right };
@@ -398,7 +409,8 @@ void ConsoleReporter::assertionEnded(AssertionStats const& _assertionStats) {
     bool includeResults = m_config->includeSuccessfulResults() || !result.isOk();
 
     // Drop out if result was successful but we're not printing them.
-    if (!includeResults && result.getResultType() != ResultWas::Warning)
+    // TODO: Make configurable whether skips should be printed
+    if (!includeResults && result.getResultType() != ResultWas::Warning && result.getResultType() != ResultWas::ExplicitSkip)
         return;
 
     lazyPrint();
@@ -641,6 +653,9 @@ void ConsoleReporter::printTotals( Totals const& totals ) {
         columns.push_back(SummaryColumn("failed as expected", Colour::ResultExpectedFailure)
                           .addRow(totals.testCases.failedButOk)
                           .addRow(totals.assertions.failedButOk));
+        columns.push_back(SummaryColumn("skipped", Colour::Skip)
+                          .addRow(totals.testCases.skipped)
+                          .addRow(totals.assertions.skipped));
 
         printSummaryRow("test cases"_sr, columns, 0);
         printSummaryRow("assertions"_sr, columns, 1);
@@ -669,17 +684,19 @@ void ConsoleReporter::printTotalsDivider(Totals const& totals) {
         std::size_t failedRatio = makeRatio(totals.testCases.failed, totals.testCases.total());
         std::size_t failedButOkRatio = makeRatio(totals.testCases.failedButOk, totals.testCases.total());
         std::size_t passedRatio = makeRatio(totals.testCases.passed, totals.testCases.total());
-        while (failedRatio + failedButOkRatio + passedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1)
-            findMax(failedRatio, failedButOkRatio, passedRatio)++;
+        std::size_t skippedRatio = makeRatio(totals.testCases.skipped, totals.testCases.total());
+        while (failedRatio + failedButOkRatio + passedRatio + skippedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1)
+            findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)++;
         while (failedRatio + failedButOkRatio + passedRatio > CATCH_CONFIG_CONSOLE_WIDTH - 1)
-            findMax(failedRatio, failedButOkRatio, passedRatio)--;
+            findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)--;
 
         m_stream << Colour(Colour::Error) << std::string(failedRatio, '=');
         m_stream << Colour(Colour::ResultExpectedFailure) << std::string(failedButOkRatio, '=');
         if (totals.testCases.allPassed())
             m_stream << Colour(Colour::ResultSuccess) << std::string(passedRatio, '=');
         else
             m_stream << Colour(Colour::Success) << std::string(passedRatio, '=');
+        m_stream << Colour(Colour::Skip) << std::string(skippedRatio, '=');
     } else {
         m_stream << Colour(Colour::Warning) << std::string(CATCH_CONFIG_CONSOLE_WIDTH - 1, '=');
     }

src/catch2/reporters/catch_reporter_junit.cpp

@@ -249,6 +249,7 @@ namespace Catch {
                 case ResultWas::Unknown:
                 case ResultWas::FailureBit:
                 case ResultWas::Exception:
+                case ResultWas::ExplicitSkip:
                     elementName = "internalError";
                     break;
             }

src/catch2/reporters/catch_reporter_sonarqube.cpp

@@ -105,6 +105,7 @@ namespace Catch {
                 case ResultWas::Unknown:
                 case ResultWas::FailureBit:
                 case ResultWas::Exception:
+                case ResultWas::ExplicitSkip:
                     elementName = "internalError";
                     break;
                 }

src/catch2/reporters/catch_reporter_tap.cpp

@@ -98,6 +98,7 @@ namespace Catch {
                 case ResultWas::Unknown:
                 case ResultWas::FailureBit:
                 case ResultWas::Exception:
+                case ResultWas::ExplicitSkip:
                     printResultType("** internal error **"_sr);
                     break;
                 }

src/catch2/reporters/catch_reporter_teamcity.cpp

@@ -88,6 +88,7 @@ namespace Catch {
             case ResultWas::Ok:
             case ResultWas::Info:
             case ResultWas::Warning:
+            case ResultWas::ExplicitSkip:
                 CATCH_ERROR("Internal error in TeamCity reporter");
                 // These cases are here to prevent compiler warnings
             case ResultWas::Unknown:

tests/CMakeLists.txt

@@ -108,6 +108,7 @@ set(TEST_SOURCES
         ${SELF_TEST_DIR}/UsageTests/Generators.tests.cpp
         ${SELF_TEST_DIR}/UsageTests/Message.tests.cpp
         ${SELF_TEST_DIR}/UsageTests/Misc.tests.cpp
+        ${SELF_TEST_DIR}/UsageTests/Skip.tests.cpp
         ${SELF_TEST_DIR}/UsageTests/ToStringByte.tests.cpp
         ${SELF_TEST_DIR}/UsageTests/ToStringChrono.tests.cpp
         ${SELF_TEST_DIR}/UsageTests/ToStringGeneral.tests.cpp

tests/SelfTest/Baselines/automake.sw.approved.txt

@@ -298,10 +298,12 @@ Message from section two
 :test-result: PASS comparisons between const int variables
 :test-result: PASS comparisons between int variables
 :test-result: PASS convertToBits
+:test-result: XFAIL dynamic skipping works with generators
 :test-result: PASS empty tags are not allowed
 :test-result: PASS erfc_inv
 :test-result: PASS estimate_clock_resolution
 :test-result: PASS even more nested SECTION tests
+:test-result: XFAIL failed assertions before SKIP are still reported
 :test-result: FAIL first tag
 loose text artifact
 :test-result: FAIL has printf
@@ -339,9 +341,11 @@ loose text artifact
 :test-result: PASS run_for_at_least, chronometer
 :test-result: PASS run_for_at_least, int
 :test-result: FAIL second tag
+:test-result: XFAIL sections can be skipped dynamically at runtime
 :test-result: FAIL send a single char to INFO
 :test-result: FAIL sends information to INFO
 :test-result: PASS shortened hide tags are split apart
+:test-result: XFAIL skipped tests can optionally provide a reason
 :test-result: PASS splitString
 :test-result: FAIL stacks unscoped info in loops
 :test-result: PASS startsWith
@@ -362,6 +366,7 @@ loose text artifact
 :test-result: PASS strlen3
 :test-result: PASS tables
 :test-result: PASS tags with dots in later positions are not parsed as hidden
+:test-result: XFAIL tests can be skipped dynamically at runtime
 :test-result: FAIL thrown std::strings are translated
 :test-result: PASS toString on const wchar_t const pointer returns the string contents
 :test-result: PASS toString on const wchar_t pointer returns the string contents

tests/SelfTest/Baselines/automake.sw.multi.approved.txt

@@ -291,10 +291,12 @@
 :test-result: PASS comparisons between const int variables
 :test-result: PASS comparisons between int variables
 :test-result: PASS convertToBits
+:test-result: XFAIL dynamic skipping works with generators
 :test-result: PASS empty tags are not allowed
 :test-result: PASS erfc_inv
 :test-result: PASS estimate_clock_resolution
 :test-result: PASS even more nested SECTION tests
+:test-result: XFAIL failed assertions before SKIP are still reported
 :test-result: FAIL first tag
 :test-result: FAIL has printf
 :test-result: PASS is_unary_function
@@ -331,9 +333,11 @@
 :test-result: PASS run_for_at_least, chronometer
 :test-result: PASS run_for_at_least, int
 :test-result: FAIL second tag
+:test-result: XFAIL sections can be skipped dynamically at runtime
 :test-result: FAIL send a single char to INFO
 :test-result: FAIL sends information to INFO
 :test-result: PASS shortened hide tags are split apart
+:test-result: XFAIL skipped tests can optionally provide a reason
 :test-result: PASS splitString
 :test-result: FAIL stacks unscoped info in loops
 :test-result: PASS startsWith
@@ -354,6 +358,7 @@
 :test-result: PASS strlen3
 :test-result: PASS tables
 :test-result: PASS tags with dots in later positions are not parsed as hidden
+:test-result: XFAIL tests can be skipped dynamically at runtime
 :test-result: FAIL thrown std::strings are translated
 :test-result: PASS toString on const wchar_t const pointer returns the string contents
 :test-result: PASS toString on const wchar_t pointer returns the string contents