Remove the new tests from approvals

They lead to stringification of file (which is ok) and file line (not ok) to the approvals, which makes them exceedingly brittle and not worth approval testing. Instead we just run them as part of the base test run.
catchorg · Aug 9, 2019 · bb5e620 · bb5e620
1 parent edcfd7f
commit bb5e620
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 10 deletions.
diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt
@@ -265,10 +265,11 @@ ToStringGeneral.tests.cpp:<line number>: passed: c == i for: 2 == 2
 ToStringGeneral.tests.cpp:<line number>: passed: c == i for: 3 == 3
 ToStringGeneral.tests.cpp:<line number>: passed: c == i for: 4 == 4
 ToStringGeneral.tests.cpp:<line number>: passed: c == i for: 5 == 5
-Details.tests.cpp:<line number>: passed: Catch::throw_logic_error((Catch::ReusableStringStream() << ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast<std::size_t>( 9 ) ) << ": Internal Catch2 error: " << "").str()), std::logic_error
+Details.tests.cpp:<line number>: passed: Catch::throw_logic_error((Catch::ReusableStringStream() << ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast<std::size_t>( 14 ) ) << ": Internal Catch2 error: " << "").str()), std::logic_error
 Details.tests.cpp:<line number>: passed: Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()), std::domain_error
 Details.tests.cpp:<line number>: passed: Catch::throw_runtime_error((Catch::ReusableStringStream() << "").str()), std::runtime_error
 Details.tests.cpp:<line number>: passed: [](){do{ if( !(false) ) Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()); } while(false);}(), std::domain_error
+Details.tests.cpp:<line number>: passed: [](){do{ if( !(true) ) Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()); } while(false);}()
 Tricky.tests.cpp:<line number>: passed: std::vector<constructor_throws>{constructor_throws{}, constructor_throws{}}
 Tricky.tests.cpp:<line number>: passed: std::vector<constructor_throws>{constructor_throws{}, constructor_throws{}}
 Tricky.tests.cpp:<line number>: passed: std::vector<int>{1, 2, 3} == std::vector<int>{1, 2, 3}

diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -1381,5 +1381,5 @@ due to unexpected exception with message:
 
 ===============================================================================
 test cases:  299 |  225 passed |  70 failed |  4 failed as expected
-assertions: 1557 | 1405 passed | 131 failed | 21 failed as expected
+assertions: 1558 | 1406 passed | 131 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -2135,7 +2135,7 @@ Details.tests.cpp:<line number>
 ...............................................................................
 
 Details.tests.cpp:<line number>: PASSED:
-  REQUIRE_THROWS_AS( Catch::throw_logic_error((Catch::ReusableStringStream() << ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast<std::size_t>( 9 ) ) << ": Internal Catch2 error: " << "").str()), std::logic_error )
+  REQUIRE_THROWS_AS( Catch::throw_logic_error((Catch::ReusableStringStream() << ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast<std::size_t>( 14 ) ) << ": Internal Catch2 error: " << "").str()), std::logic_error )
 
 Details.tests.cpp:<line number>: PASSED:
   REQUIRE_THROWS_AS( Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()), std::domain_error )
@@ -2146,6 +2146,9 @@ Details.tests.cpp:<line number>: PASSED:
 Details.tests.cpp:<line number>: PASSED:
   REQUIRE_THROWS_AS( [](){do{ if( !(false) ) Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()); } while(false);}(), std::domain_error )
 
+Details.tests.cpp:<line number>: PASSED:
+  REQUIRE_NOTHROW( [](){do{ if( !(true) ) Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()); } while(false);}() )
+
 -------------------------------------------------------------------------------
 Commas in various macros are allowed
 -------------------------------------------------------------------------------
@@ -12428,5 +12431,5 @@ Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
 test cases:  299 |  209 passed |  86 failed |  4 failed as expected
-assertions: 1574 | 1405 passed | 148 failed | 21 failed as expected
+assertions: 1575 | 1406 passed | 148 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="132" tests="1575" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="132" tests="1576" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
       <property name="random-seed" value="1"/>

diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -2441,10 +2441,10 @@ Nor would this
     <TestCase name="Check that our error handling macros throw the right exceptions" tags="[!throws][internals]" filename="projects/<exe-name>/IntrospectiveTests/Details.tests.cpp" >
       <Expression success="true" type="REQUIRE_THROWS_AS" filename="projects/<exe-name>/IntrospectiveTests/Details.tests.cpp" >
         <Original>
-          Catch::throw_logic_error((Catch::ReusableStringStream() &lt;&lt; ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast&lt;std::size_t>( 9 ) ) &lt;&lt; ": Internal Catch2 error: " &lt;&lt; "").str()), std::logic_error
+          Catch::throw_logic_error((Catch::ReusableStringStream() &lt;&lt; ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast&lt;std::size_t>( 14 ) ) &lt;&lt; ": Internal Catch2 error: " &lt;&lt; "").str()), std::logic_error
         </Original>
         <Expanded>
-          Catch::throw_logic_error((Catch::ReusableStringStream() &lt;&lt; ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast&lt;std::size_t>( 9 ) ) &lt;&lt; ": Internal Catch2 error: " &lt;&lt; "").str()), std::logic_error
+          Catch::throw_logic_error((Catch::ReusableStringStream() &lt;&lt; ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast&lt;std::size_t>( 14 ) ) &lt;&lt; ": Internal Catch2 error: " &lt;&lt; "").str()), std::logic_error
         </Expanded>
       </Expression>
       <Expression success="true" type="REQUIRE_THROWS_AS" filename="projects/<exe-name>/IntrospectiveTests/Details.tests.cpp" >
@@ -2471,6 +2471,14 @@ Nor would this
           [](){do{ if( !(false) ) Catch::throw_domain_error((Catch::ReusableStringStream() &lt;&lt; "").str()); } while(false);}(), std::domain_error
         </Expanded>
       </Expression>
+      <Expression success="true" type="REQUIRE_NOTHROW" filename="projects/<exe-name>/IntrospectiveTests/Details.tests.cpp" >
+        <Original>
+          [](){do{ if( !(true) ) Catch::throw_domain_error((Catch::ReusableStringStream() &lt;&lt; "").str()); } while(false);}()
+        </Original>
+        <Expanded>
+          [](){do{ if( !(true) ) Catch::throw_domain_error((Catch::ReusableStringStream() &lt;&lt; "").str()); } while(false);}()
+        </Expanded>
+      </Expression>
       <OverallResult success="true"/>
     </TestCase>
     <TestCase name="Commas in various macros are allowed" filename="projects/<exe-name>/UsageTests/Tricky.tests.cpp" >
@@ -14802,7 +14810,7 @@ loose text artifact
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="1405" failures="149" expectedFailures="21"/>
+    <OverallResults successes="1406" failures="149" expectedFailures="21"/>
   </Group>
-  <OverallResults successes="1405" failures="148" expectedFailures="21"/>
+  <OverallResults successes="1406" failures="148" expectedFailures="21"/>
 </Catch>
diff --git a/projects/SelfTest/IntrospectiveTests/Details.tests.cpp b/projects/SelfTest/IntrospectiveTests/Details.tests.cpp
@@ -10,7 +10,7 @@
 #pragma warning(disable:4702) // unreachable code in the macro expansions
 #endif
 
-TEST_CASE("Check that our error handling macros throw the right exceptions", "[!throws][internals]") {
+TEST_CASE("Check that our error handling macros throw the right exceptions", "[!throws][internals][approvals]") {
     REQUIRE_THROWS_AS(CATCH_INTERNAL_ERROR(""), std::logic_error);
     REQUIRE_THROWS_AS(CATCH_ERROR(""), std::domain_error);
     REQUIRE_THROWS_AS(CATCH_RUNTIME_ERROR(""), std::runtime_error);