From 3173df87ed70586e86873db99bf8e885d41fd830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Je=C5=99=C3=A1bek?= Date: Fri, 11 Jun 2021 10:23:18 +0200 Subject: [PATCH] WIP --- docs/command-line.md | 8 ++++---- docs/reporters.md | 4 ++-- src/catch2/catch_config.cpp | 20 ++++++++++++-------- src/catch2/catch_config.hpp | 4 +++- tools/scripts/approvalTests.py | 26 ++++++++------------------ 5 files changed, 29 insertions(+), 33 deletions(-) diff --git a/docs/command-line.md b/docs/command-line.md index a3db375f92..54f5101e07 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -175,7 +175,7 @@ If one or more test-specs have been supplied too then only the matching tests wi ## Sending output to a file -
-o, --out <filename>
+
-o, --out <filename>
 
Use this option to send all output to a file. By default output is sent to stdout (note that uses of stdout and stderr *from within test cases* are redirected and included in the report - so even stderr will effectively end up on stdout). @@ -281,7 +281,7 @@ is that as long as the random seed is fixed, running only some tests ## Specify a seed for the Random Number Generator
--rng-seed <'time'|number>
-Sets a seed for the random number generator using ```std::srand()```. +Sets a seed for the random number generator using ```std::srand()```. If a number is provided this is used directly as the seed so the random pattern is repeatable. Alternatively if the keyword ```time``` is provided then the result of calling ```std::time(0)``` is used and so the pattern becomes unpredictable. In some cases, you might need to pass the keyword ```time``` in double quotes instead of single quotes. @@ -396,7 +396,7 @@ There are some limitations of this feature to be aware of: - Code outside of sections being skipped will still be executed - e.g. any set-up code in the TEST_CASE before the start of the first section.
- At time of writing, wildcards are not supported in section names. -- If you specify a section without narrowing to a test case first then all test cases will be executed +- If you specify a section without narrowing to a test case first then all test cases will be executed (but only matching sections within them). @@ -404,7 +404,7 @@ start of the first section.
## Filenames as tags
-#, --filenames-as-tags
-When this option is used then every test is given an additional tag which is formed of the unqualified +When this option is used then every test is given an additional tag which is formed of the unqualified filename it is found in, with any extension stripped, prefixed with the `#` character. So, for example, tests within the file `~\Dev\MyProject\Ferrets.cpp` would be tagged `[#Ferrets]`. diff --git a/docs/reporters.md b/docs/reporters.md index 7f521bfc4e..25f686b0e5 100644 --- a/docs/reporters.md +++ b/docs/reporters.md @@ -19,13 +19,13 @@ There are four reporters built in to the single include: * `console` writes as lines of text, formatted to a typical terminal width, with colours if a capable terminal is detected. * `compact` similar to `console` but optimised for minimal output - each entry on one line * `junit` writes xml that corresponds to Ant's [junitreport](http://help.catchsoftware.com/display/ET/JUnit+Format) target. Useful for build systems that understand Junit. -Because of the way the junit format is structured the run must complete before anything is written. +Because of the way the junit format is structured the run must complete before anything is written. * `xml` writes an xml format tailored to Catch. Unlike `junit` this is a streaming format so results are delivered progressively. There are a few additional reporters, for specific build systems, in the Catch repository (in `include\reporters`) which you can `#include` in your project if you would like to make use of them. Do this in one source file - the same one you have `CATCH_CONFIG_MAIN` or `CATCH_CONFIG_RUNNER`. -* `teamcity` writes the native, streaming, format that [TeamCity](https://www.jetbrains.com/teamcity/) understands. +* `teamcity` writes the native, streaming, format that [TeamCity](https://www.jetbrains.com/teamcity/) understands. Use this when building as part of a TeamCity build to see results as they happen ([code example](../examples/207-Rpt-TeamCityReporter.cpp)). * `tap` writes in the TAP ([Test Anything Protocol](https://en.wikipedia.org/wiki/Test_Anything_Protocol)) format. * `automake` writes in a format that correspond to [automake .trs](https://www.gnu.org/software/automake/manual/html_node/Log-files-generation-and-test-results-recording.html) files diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp index 28caf8c38a..b08f8ef7b8 100644 --- a/src/catch2/catch_config.cpp +++ b/src/catch2/catch_config.cpp @@ -20,16 +20,18 @@ namespace Catch { namespace { class RDBufStream : public IStream { mutable std::ostream m_os; + public: - // Note that the streambuf must live at least as long as this object. - RDBufStream(std::streambuf *sb) : m_os( sb ) {} + // Note that the streambuf must live at least as long as this + // object. + RDBufStream( std::streambuf* sb ): m_os( sb ) {} ~RDBufStream() override = default; public: // IStream std::ostream& stream() const override { return m_os; } }; } // unnamed namespace - } // namespace Detail + } // namespace Detail std::ostream& operator<<( std::ostream& os, ConfigData::ReporterAndFile const& reporter ) { @@ -63,12 +65,14 @@ namespace Catch { } m_testSpec = parser.testSpec(); - m_reporterStreams.reserve(m_data.reporterSpecifications.size()); - for (auto const& reporterAndFile : m_data.reporterSpecifications) { - if (reporterAndFile.outputFileName.none()) { - m_reporterStreams.emplace_back(new Detail::RDBufStream(m_defaultStream->stream().rdbuf())); + m_reporterStreams.reserve( m_data.reporterSpecifications.size() ); + for ( auto const& reporterAndFile : m_data.reporterSpecifications ) { + if ( reporterAndFile.outputFileName.none() ) { + m_reporterStreams.emplace_back( new Detail::RDBufStream( + m_defaultStream->stream().rdbuf() ) ); } else { - m_reporterStreams.emplace_back(openStream(*reporterAndFile.outputFileName)); + m_reporterStreams.emplace_back( + openStream( *reporterAndFile.outputFileName ) ); } } } diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index 5f0a608f96..52f5def01b 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -69,7 +69,9 @@ namespace Catch { #ifndef CATCH_CONFIG_DEFAULT_REPORTER #define CATCH_CONFIG_DEFAULT_REPORTER "console" #endif - std::vector reporterSpecifications = { {CATCH_CONFIG_DEFAULT_REPORTER, {}} }; + std::vector reporterSpecifications = { + {CATCH_CONFIG_DEFAULT_REPORTER, {}} + }; // Internal: used as parser state bool _nonDefaultReporterSpecifications = false; #undef CATCH_CONFIG_DEFAULT_REPORTER diff --git a/tools/scripts/approvalTests.py b/tools/scripts/approvalTests.py index 04b25e99af..b38c657dfc 100755 --- a/tools/scripts/approvalTests.py +++ b/tools/scripts/approvalTests.py @@ -178,8 +178,12 @@ def run_test(baseName, args): f.close() -def check_outputs(baseName, baselinesPath, rawResultsPath, filteredResultsPath): +def check_outputs(baseName): global overallResult + rawResultsPath = get_rawResultsPath(baseName) + baselinesPath = get_baselinesPath(baseName) + filteredResultsPath = get_filteredResultsPath(baseName) + rawFile = io.open(rawResultsPath, 'r', encoding='utf-8', errors='surrogateescape') filteredFile = io.open(filteredResultsPath, 'w', encoding='utf-8', errors='surrogateescape') for line in rawFile: @@ -208,12 +212,8 @@ def check_outputs(baseName, baselinesPath, rawResultsPath, filteredResultsPath): def approve(baseName, args): - baselinesPath = get_baselinesPath(baseName) - rawResultsPath = get_rawResultsPath(baseName) - filteredResultsPath = get_filteredResultsPath(baseName) - run_test(baseName, args) - check_outputs(baseName, baselinesPath, rawResultsPath, filteredResultsPath) + check_outputs(baseName) print("Running approvals against executable:") @@ -243,19 +243,9 @@ def approve(baseName, args): reporter_args += ['-r', '{}:{}'.format(reporter, get_rawResultsPath(filename))] run_test("default.sw.multi", common_args + reporter_args) -check_outputs( - "default.sw.multi", - get_baselinesPath("default.sw.multi"), - get_rawResultsPath("default.sw.multi"), - get_filteredResultsPath("default.sw.multi") -) +check_outputs("default.sw.multi") for reporter, filename in zip(reporters, filenames): - check_outputs( - filename, - get_baselinesPath(filename), - get_rawResultsPath(filename), - get_filteredResultsPath(filename) - ) + check_outputs(filename) if overallResult != 0: