Remove the new tests from approvals

They lead to stringification of file (which is ok) and file line (not ok) to the approvals, which makes them exceedingly brittle and not worth approval testing. Instead we just run them as part of the base test run.
catchorg · Aug 9, 2019 · ebe6a07 · ebe6a07
1 parent edcfd7f
commit ebe6a07
Show file tree

Hide file tree

Showing 6 changed files with 8 additions and 66 deletions.
diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt
@@ -265,10 +265,6 @@ ToStringGeneral.tests.cpp:<line number>: passed: c == i for: 2 == 2
 ToStringGeneral.tests.cpp:<line number>: passed: c == i for: 3 == 3
 ToStringGeneral.tests.cpp:<line number>: passed: c == i for: 4 == 4
 ToStringGeneral.tests.cpp:<line number>: passed: c == i for: 5 == 5
-Details.tests.cpp:<line number>: passed: Catch::throw_logic_error((Catch::ReusableStringStream() << ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast<std::size_t>( 9 ) ) << ": Internal Catch2 error: " << "").str()), std::logic_error
-Details.tests.cpp:<line number>: passed: Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()), std::domain_error
-Details.tests.cpp:<line number>: passed: Catch::throw_runtime_error((Catch::ReusableStringStream() << "").str()), std::runtime_error
-Details.tests.cpp:<line number>: passed: [](){do{ if( !(false) ) Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()); } while(false);}(), std::domain_error
 Tricky.tests.cpp:<line number>: passed: std::vector<constructor_throws>{constructor_throws{}, constructor_throws{}}
 Tricky.tests.cpp:<line number>: passed: std::vector<constructor_throws>{constructor_throws{}, constructor_throws{}}
 Tricky.tests.cpp:<line number>: passed: std::vector<int>{1, 2, 3} == std::vector<int>{1, 2, 3}

diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -1380,6 +1380,6 @@ due to unexpected exception with message:
   Why would you throw a std::string?
 
 ===============================================================================
-test cases:  299 |  225 passed |  70 failed |  4 failed as expected
-assertions: 1557 | 1405 passed | 131 failed | 21 failed as expected
+test cases:  298 |  224 passed |  70 failed |  4 failed as expected
+assertions: 1553 | 1401 passed | 131 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -2128,24 +2128,6 @@ ToStringGeneral.tests.cpp:<line number>: PASSED:
 with expansion:
   5 == 5
 
--------------------------------------------------------------------------------
-Check that our error handling macros throw the right exceptions
--------------------------------------------------------------------------------
-Details.tests.cpp:<line number>
-...............................................................................
-
-Details.tests.cpp:<line number>: PASSED:
-  REQUIRE_THROWS_AS( Catch::throw_logic_error((Catch::ReusableStringStream() << ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast<std::size_t>( 9 ) ) << ": Internal Catch2 error: " << "").str()), std::logic_error )
-
-Details.tests.cpp:<line number>: PASSED:
-  REQUIRE_THROWS_AS( Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()), std::domain_error )
-
-Details.tests.cpp:<line number>: PASSED:
-  REQUIRE_THROWS_AS( Catch::throw_runtime_error((Catch::ReusableStringStream() << "").str()), std::runtime_error )
-
-Details.tests.cpp:<line number>: PASSED:
-  REQUIRE_THROWS_AS( [](){do{ if( !(false) ) Catch::throw_domain_error((Catch::ReusableStringStream() << "").str()); } while(false);}(), std::domain_error )
-
 -------------------------------------------------------------------------------
 Commas in various macros are allowed
 -------------------------------------------------------------------------------
@@ -12427,6 +12409,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
-test cases:  299 |  209 passed |  86 failed |  4 failed as expected
-assertions: 1574 | 1405 passed | 148 failed | 21 failed as expected
+test cases:  298 |  208 passed |  86 failed |  4 failed as expected
+assertions: 1570 | 1401 passed | 148 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="132" tests="1575" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="132" tests="1571" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
       <property name="random-seed" value="1"/>
@@ -241,7 +241,6 @@ Exception.tests.cpp:<line number>
     <testcase classname="<exe-name>.global" name="Character pretty printing/Specifically escaped" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Character pretty printing/General chars" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Character pretty printing/Low ASCII" time="{duration}"/>
-    <testcase classname="<exe-name>.global" name="Check that our error handling macros throw the right exceptions" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Commas in various macros are allowed" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Comparing function pointers" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Comparison with explicitly convertible types" time="{duration}"/>

diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -2438,41 +2438,6 @@ Nor would this
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <TestCase name="Check that our error handling macros throw the right exceptions" tags="[!throws][internals]" filename="projects/<exe-name>/IntrospectiveTests/Details.tests.cpp" >
-      <Expression success="true" type="REQUIRE_THROWS_AS" filename="projects/<exe-name>/IntrospectiveTests/Details.tests.cpp" >
-        <Original>
-          Catch::throw_logic_error((Catch::ReusableStringStream() &lt;&lt; ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast&lt;std::size_t>( 9 ) ) &lt;&lt; ": Internal Catch2 error: " &lt;&lt; "").str()), std::logic_error
-        </Original>
-        <Expanded>
-          Catch::throw_logic_error((Catch::ReusableStringStream() &lt;&lt; ::Catch::SourceLineInfo( "projects/<exe-name>/IntrospectiveTests/Details.tests.cpp", static_cast&lt;std::size_t>( 9 ) ) &lt;&lt; ": Internal Catch2 error: " &lt;&lt; "").str()), std::logic_error
-        </Expanded>
-      </Expression>
-      <Expression success="true" type="REQUIRE_THROWS_AS" filename="projects/<exe-name>/IntrospectiveTests/Details.tests.cpp" >
-        <Original>
-          Catch::throw_domain_error((Catch::ReusableStringStream() &lt;&lt; "").str()), std::domain_error
-        </Original>
-        <Expanded>
-          Catch::throw_domain_error((Catch::ReusableStringStream() &lt;&lt; "").str()), std::domain_error
-        </Expanded>
-      </Expression>
-      <Expression success="true" type="REQUIRE_THROWS_AS" filename="projects/<exe-name>/IntrospectiveTests/Details.tests.cpp" >
-        <Original>
-          Catch::throw_runtime_error((Catch::ReusableStringStream() &lt;&lt; "").str()), std::runtime_error
-        </Original>
-        <Expanded>
-          Catch::throw_runtime_error((Catch::ReusableStringStream() &lt;&lt; "").str()), std::runtime_error
-        </Expanded>
-      </Expression>
-      <Expression success="true" type="REQUIRE_THROWS_AS" filename="projects/<exe-name>/IntrospectiveTests/Details.tests.cpp" >
-        <Original>
-          [](){do{ if( !(false) ) Catch::throw_domain_error((Catch::ReusableStringStream() &lt;&lt; "").str()); } while(false);}(), std::domain_error
-        </Original>
-        <Expanded>
-          [](){do{ if( !(false) ) Catch::throw_domain_error((Catch::ReusableStringStream() &lt;&lt; "").str()); } while(false);}(), std::domain_error
-        </Expanded>
-      </Expression>
-      <OverallResult success="true"/>
-    </TestCase>
     <TestCase name="Commas in various macros are allowed" filename="projects/<exe-name>/UsageTests/Tricky.tests.cpp" >
       <Expression success="true" type="REQUIRE_THROWS" filename="projects/<exe-name>/UsageTests/Tricky.tests.cpp" >
         <Original>
@@ -14802,7 +14767,7 @@ loose text artifact
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="1405" failures="149" expectedFailures="21"/>
+    <OverallResults successes="1401" failures="149" expectedFailures="21"/>
   </Group>
-  <OverallResults successes="1405" failures="148" expectedFailures="21"/>
+  <OverallResults successes="1401" failures="148" expectedFailures="21"/>
 </Catch>
diff --git a/projects/SelfTest/IntrospectiveTests/Details.tests.cpp b/projects/SelfTest/IntrospectiveTests/Details.tests.cpp
@@ -10,7 +10,7 @@
 #pragma warning(disable:4702) // unreachable code in the macro expansions
 #endif
 
-TEST_CASE("Check that our error handling macros throw the right exceptions", "[!throws][internals]") {
+TEST_CASE("Check that our error handling macros throw the right exceptions", "[!throws][internals][approvals]") {
     REQUIRE_THROWS_AS(CATCH_INTERNAL_ERROR(""), std::logic_error);
     REQUIRE_THROWS_AS(CATCH_ERROR(""), std::domain_error);
     REQUIRE_THROWS_AS(CATCH_RUNTIME_ERROR(""), std::runtime_error);