Always report rng seed from builtin reporters

Not all reporters use a format that supports this, so TeamCity and Automake reporters still do not report it. The console reporter now reports it even on successful runs, where before it only reported the rng seed in the header, which was showed either for failed run, or for run with `-s`. CLoses #2065
catchorg · Apr 19, 2022 · 17a04f8 · 17a04f8
1 parent 90e6905
commit 17a04f8
Show file tree

Hide file tree

Showing 19 changed files with 73 additions and 20 deletions.
diff --git a/src/catch2/reporters/catch_reporter_compact.cpp b/src/catch2/reporters/catch_reporter_compact.cpp
@@ -7,6 +7,7 @@
 // SPDX-License-Identifier: BSL-1.0
 #include <catch2/reporters/catch_reporter_compact.hpp>
 
+#include <catch2/catch_test_spec.hpp>
 #include <catch2/reporters/catch_reporter_helpers.hpp>
 #include <catch2/interfaces/catch_interfaces_config.hpp>
 #include <catch2/internal/catch_platform.hpp>
@@ -253,6 +254,16 @@ class AssertionPrinter {
             m_stream << "No test cases matched '" << unmatchedSpec << "'\n";
         }
 
+        void CompactReporter::testRunStarting( TestRunInfo const& ) {
+            if ( m_config->testSpec().hasFilters() ) {
+                m_stream << m_colour->guardColour( Colour::BrightYellow )
+                         << "Filters: "
+                         << serializeFilters( m_config->getTestsOrTags() )
+                         << '\n';
+            }
+            m_stream << "RNG seed: " << m_config->rngSeed() << '\n';
+        }
+
         void CompactReporter::assertionEnded( AssertionStats const& _assertionStats ) {
             AssertionResult const& result = _assertionStats.assertionResult;
 

diff --git a/src/catch2/reporters/catch_reporter_compact.hpp b/src/catch2/reporters/catch_reporter_compact.hpp
@@ -24,6 +24,8 @@ namespace Catch {
 
         void noMatchingTestCases( StringRef unmatchedSpec ) override;
 
+        void testRunStarting( TestRunInfo const& _testInfo ) override;
+
         void assertionEnded(AssertionStats const& _assertionStats) override;
 
         void sectionEnded(SectionStats const& _sectionStats) override;

diff --git a/src/catch2/reporters/catch_reporter_console.cpp b/src/catch2/reporters/catch_reporter_console.cpp
@@ -496,7 +496,11 @@ void ConsoleReporter::testRunEnded(TestRunStats const& _testRunStats) {
 }
 void ConsoleReporter::testRunStarting(TestRunInfo const& _testInfo) {
     StreamingReporterBase::testRunStarting(_testInfo);
-    printTestFilters();
+    if ( m_config->testSpec().hasFilters() ) {
+        m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: "
+                 << serializeFilters( m_config->getTestsOrTags() ) << '\n';
+    }
+    m_stream << "Randomness seeded to: " << m_config->rngSeed() << '\n';
 }
 
 void ConsoleReporter::lazyPrint() {
@@ -521,8 +525,7 @@ void ConsoleReporter::lazyPrintRunInfo() {
              << m_colour->guardColour( Colour::SecondaryText )
              << currentTestRunInfo.name << " is a Catch2 v" << libraryVersion()
              << " host application.\n"
-             << "Run with -? for options\n\n"
-             << "Randomness seeded to: " << m_config->rngSeed() << "\n\n";
+             << "Run with -? for options\n\n";
 
     m_testRunInfoPrinted = true;
 }
@@ -701,13 +704,6 @@ void ConsoleReporter::printSummaryDivider() {
     m_stream << lineOfChars('-') << '\n';
 }
 
-void ConsoleReporter::printTestFilters() {
-    if (m_config->testSpec().hasFilters()) {
-        m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: "
-                 << serializeFilters( m_config->getTestsOrTags() ) << '\n';
-    }
-}
-
 } // end namespace Catch
 
 #if defined(_MSC_VER)

diff --git a/src/catch2/reporters/catch_reporter_console.hpp b/src/catch2/reporters/catch_reporter_console.hpp
@@ -63,7 +63,6 @@ namespace Catch {
 
         void printTotalsDivider(Totals const& totals);
         void printSummaryDivider();
-        void printTestFilters();
 
         bool m_headerPrinted = false;
         bool m_testRunInfoPrinted = false;

diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp
@@ -9,13 +9,25 @@
 
 #include <catch2/internal/catch_string_manip.hpp>
 #include <catch2/catch_test_case_info.hpp>
+#include <catch2/internal/catch_reusable_string_stream.hpp>
+#include <catch2/interfaces/catch_interfaces_config.hpp>
 
 #include <map>
 
 namespace Catch {
 
+    namespace {
+        std::string createRngSeedString(uint32_t seed) {
+            ReusableStringStream sstr;
+            sstr << "rng-seed=" << seed;
+            return sstr.str();
+        }
+    }
+
     void SonarQubeReporter::testRunStarting(TestRunInfo const& testRunInfo) {
         CumulativeReporterBase::testRunStarting(testRunInfo);
+
+        xml.writeComment( createRngSeedString( m_config->rngSeed() ) );
         xml.startElement("testExecutions");
         xml.writeAttribute("version"_sr, '1');
     }

diff --git a/src/catch2/reporters/catch_reporter_tap.cpp b/src/catch2/reporters/catch_reporter_tap.cpp
@@ -9,6 +9,7 @@
 #include <catch2/internal/catch_console_colour.hpp>
 #include <catch2/internal/catch_string_manip.hpp>
 #include <catch2/catch_test_case_info.hpp>
+#include <catch2/interfaces/catch_interfaces_config.hpp>
 
 #include <algorithm>
 #include <iterator>
@@ -189,6 +190,10 @@ namespace Catch {
 
     } // End anonymous namespace
 
+    void TAPReporter::testRunStarting( TestRunInfo const& ) {
+        m_stream << "# rng-seed: " << m_config->rngSeed() << '\n';
+    }
+
     void TAPReporter::noMatchingTestCases( StringRef unmatchedSpec ) {
         m_stream << "# No test cases matched '" << unmatchedSpec << "'\n";
     }

diff --git a/src/catch2/reporters/catch_reporter_tap.hpp b/src/catch2/reporters/catch_reporter_tap.hpp
@@ -26,6 +26,8 @@ namespace Catch {
             return "Reports test results in TAP format, suitable for test harnesses"s;
         }
 
+        void testRunStarting( TestRunInfo const& testInfo ) override;
+
         void noMatchingTestCases( StringRef unmatchedSpec ) override;
 
         void assertionEnded(AssertionStats const& _assertionStats) override;

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -523,5 +523,27 @@ if (CATCH_ENABLE_CONFIGURE_TESTS)
     endforeach()
 endif()
 
+foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata
+                      "compact"
+                      "console"
+                      "JUnit"
+                      "SonarQube"
+                      "TAP"
+                      # "TeamCity" - does not seem to support test suite-level metadata/comments
+                      "XML")
+    add_test(NAME "Reporters:RngSeed:${reporterName}"
+      COMMAND
+        $<TARGET_FILE:SelfTest> "Factorials are computed"
+          --reporter ${reporterName}
+          --rng-seed 18181818
+    )
+    set_tests_properties("Reporters:RngSeed:${reporterName}"
+      PROPERTIES
+        PASS_REGULAR_EXPRESSION "18181818"
+    )
+
+endforeach()
+
+
 list(APPEND CATCH_WARNING_TARGETS SelfTest)
 set(CATCH_WARNING_TARGETS ${CATCH_WARNING_TARGETS} PARENT_SCOPE)
diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt
@@ -1,3 +1,5 @@
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+RNG seed: 1
 Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
 Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
 Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0

diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
@@ -1,3 +1,5 @@
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+RNG seed: 1
 Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
 Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
 Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0

diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt
@@ -1,11 +1,10 @@
 Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 <exe-name> is a <version> host application.
 Run with -? for options
 
-Randomness seeded to: 1
-
 -------------------------------------------------------------------------------
 #1455 - INFO and WARN can start with a linebreak
 -------------------------------------------------------------------------------

diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt
@@ -1,11 +1,10 @@
 Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 <exe-name> is a <version> host application.
 Run with -? for options
 
-Randomness seeded to: 1
-
 -------------------------------------------------------------------------------
 # A test name that starts with a #
 -------------------------------------------------------------------------------

diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt
@@ -1,11 +1,10 @@
 Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 <exe-name> is a <version> host application.
 Run with -? for options
 
-Randomness seeded to: 1
-
 -------------------------------------------------------------------------------
 # A test name that starts with a #
 -------------------------------------------------------------------------------

diff --git a/tests/SelfTest/Baselines/console.swa4.approved.txt b/tests/SelfTest/Baselines/console.swa4.approved.txt
@@ -1,11 +1,10 @@
 Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 <exe-name> is a <version> host application.
 Run with -? for options
 
-Randomness seeded to: 1
-
 -------------------------------------------------------------------------------
 # A test name that starts with a #
 -------------------------------------------------------------------------------

diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- rng-seed=1 -->
 <testExecutions version="1"loose text artifact
 >
   <file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">

diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- rng-seed=1 -->
 <testExecutions version="1">
   <file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">
     <testCase name="Clara::Arg supports single-arg parse the way Opt does" duration="{duration}"/>

diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt
@@ -1,3 +1,4 @@
+# rng-seed: 1
 # # A test name that starts with a #
 ok {test-number} - with 1 message: 'yay'
 # #1027: Bitfields can be captured

diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt
@@ -1,3 +1,4 @@
+# rng-seed: 1
 # # A test name that starts with a #
 ok {test-number} - with 1 message: 'yay'
 # #1027: Bitfields can be captured

diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py
@@ -38,7 +38,7 @@
 # This matches the summary line made by compact reporter, console reporter's
 # summary line does not match the regex.
 summary_regex = 'Passed \d+ test case with \d+ assertions.'
-if not re.match(summary_regex, stdout):
+if not re.search(summary_regex, stdout):
     print("Could not find '{}' in the stdout".format(summary_regex))
     print('stdout: "{}"'.format(stdout))
     exit(2)