From 291b35b389b8e6d7b560cf7e43b6c72c4b73e522 Mon Sep 17 00:00:00 2001 From: Daniel Feist Date: Sat, 23 Apr 2022 23:14:49 +0200 Subject: [PATCH] Added --skip-benchmarks command-line option. (#2408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2392 Co-authored-by: Martin Hořeňovský --- docs/command-line.md | 12 ++++++++++++ src/catch2/benchmark/catch_benchmark.hpp | 7 +++++-- src/catch2/catch_config.cpp | 1 + src/catch2/catch_config.hpp | 2 ++ .../interfaces/catch_interfaces_config.hpp | 1 + src/catch2/internal/catch_commandline.cpp | 3 +++ tests/CMakeLists.txt | 12 ++++++++++++ tests/SelfTest/UsageTests/Benchmark.tests.cpp | 17 +++++++++++++++++ 8 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/command-line.md b/docs/command-line.md index f8ac6374b8..0faeee16f9 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -19,6 +19,7 @@ [Specify a seed for the Random Number Generator](#specify-a-seed-for-the-random-number-generator)
[Identify framework and version according to the libIdentify standard](#identify-framework-and-version-according-to-the-libidentify-standard)
[Wait for key before continuing](#wait-for-key-before-continuing)
+[Skip benchmarks](#skip-benchmarks)
[Specify the number of benchmark samples to collect](#specify-the-number-of-benchmark-samples-to-collect)
[Specify the number of resamples for bootstrapping](#specify-the-number-of-resamples-for-bootstrapping)
[Specify the confidence-interval for bootstrapping](#specify-the-confidence-interval-for-bootstrapping)
@@ -62,6 +63,7 @@ Click one of the following links to take you straight to that option - or scroll ` --rng-seed`
` --libidentify`
` --wait-for-keypress`
+ ` --skip-benchmarks`
` --benchmark-samples`
` --benchmark-resamples`
` --benchmark-confidence-interval`
@@ -370,6 +372,16 @@ See [The LibIdentify repo for more information and examples](https://github.com/ Will cause the executable to print a message and wait until the return/ enter key is pressed before continuing - either before running any tests, after running all tests - or both, depending on the argument. + +## Skip all benchmarks +
--skip-benchmarks
+ +> [Introduced](https://github.com/catchorg/Catch2/issues/2408) in Catch X.Y.Z. + +This flag tells Catch2 to skip running all benchmarks. Benchmarks in this +case mean code blocks in `BENCHMARK` and `BENCHMARK_ADVANCED` macros, not +test cases with the `[!benchmark]` tag. + ## Specify the number of benchmark samples to collect 
--benchmark-samples <# of samples>
diff --git a/src/catch2/benchmark/catch_benchmark.hpp b/src/catch2/benchmark/catch_benchmark.hpp index 28664b613e..66e3670f60 100644 --- a/src/catch2/benchmark/catch_benchmark.hpp +++ b/src/catch2/benchmark/catch_benchmark.hpp @@ -95,8 +95,11 @@ namespace Catch { // sets lambda to be used in fun *and* executes benchmark! template ::value, int> = 0> Benchmark & operator=(Fun func) { - fun = Detail::BenchmarkFunction(func); - run(); + auto const* cfg = getCurrentContext().getConfig(); + if (!cfg->skipBenchmarks()) { + fun = Detail::BenchmarkFunction(func); + run(); + } return *this; } diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp index 9e56e9fe70..e3ce6b65c0 100644 --- a/src/catch2/catch_config.cpp +++ b/src/catch2/catch_config.cpp @@ -152,6 +152,7 @@ namespace Catch { bool Config::showInvisibles() const { return m_data.showInvisibles; } Verbosity Config::verbosity() const { return m_data.verbosity; } + bool Config::skipBenchmarks() const { return m_data.skipBenchmarks; } bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; } unsigned int Config::benchmarkSamples() const { return m_data.benchmarkSamples; } double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; } diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index 8823accdf2..a3dc010b70 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -64,6 +64,7 @@ namespace Catch { unsigned int shardCount = 1; unsigned int shardIndex = 0; + bool skipBenchmarks = false; bool benchmarkNoAnalysis = false; unsigned int benchmarkSamples = 100; double benchmarkConfidenceInterval = 0.95; @@ -129,6 +130,7 @@ namespace Catch { int abortAfter() const override; bool showInvisibles() const override; Verbosity verbosity() const override; + bool skipBenchmarks() const override; bool benchmarkNoAnalysis() const override; unsigned int benchmarkSamples() const override; double benchmarkConfidenceInterval() const override; diff --git a/src/catch2/interfaces/catch_interfaces_config.hpp b/src/catch2/interfaces/catch_interfaces_config.hpp index 7e3c9d0b81..5e358650a4 100644 --- a/src/catch2/interfaces/catch_interfaces_config.hpp +++ b/src/catch2/interfaces/catch_interfaces_config.hpp @@ -88,6 +88,7 @@ namespace Catch { virtual std::vector const& getSectionsToRun() const = 0; virtual Verbosity verbosity() const = 0; + virtual bool skipBenchmarks() const = 0; virtual bool benchmarkNoAnalysis() const = 0; virtual unsigned int benchmarkSamples() const = 0; virtual double benchmarkConfidenceInterval() const = 0; diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp index 59b06de9f3..f2f638bc17 100644 --- a/src/catch2/internal/catch_commandline.cpp +++ b/src/catch2/internal/catch_commandline.cpp @@ -296,6 +296,9 @@ namespace Catch { | Opt( setWaitForKeypress, "never|start|exit|both" ) ["--wait-for-keypress"] ( "waits for a keypress before exiting" ) + | Opt( config.skipBenchmarks) + ["--skip-benchmarks"] + ( "disable running benchmarks") | Opt( config.benchmarkSamples, "samples" ) ["--benchmark-samples"] ( "number of samples to collect (default: 100)" ) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 911f385710..627928f154 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -359,6 +359,18 @@ add_test(NAME CheckConvenienceHeaders ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tools/scripts/checkConvenienceHeaders.py ) +add_test(NAME "Benchmarking::SkipBenchmarkMacros" + COMMAND + $ "Skip benchmark macros" + --reporter console + --skip-benchmarks +) +set_tests_properties("Benchmarking::SkipBenchmarkMacros" + PROPERTIES + PASS_REGULAR_EXPRESSION "All tests passed \\(2 assertions in 1 test case\\)" + FAIL_REGULAR_EXPRESSION "benchmark name" +) + add_test(NAME "Benchmarking::FailureReporting::OptimizedOut" COMMAND diff --git a/tests/SelfTest/UsageTests/Benchmark.tests.cpp b/tests/SelfTest/UsageTests/Benchmark.tests.cpp index 14b001079a..ffedc9a2f6 100644 --- a/tests/SelfTest/UsageTests/Benchmark.tests.cpp +++ b/tests/SelfTest/UsageTests/Benchmark.tests.cpp @@ -152,3 +152,20 @@ TEST_CASE("Benchmark containers", "[!benchmark]") { }; } } + +TEST_CASE("Skip benchmark macros", "[!benchmark]") { + std::vector v; + BENCHMARK("fill vector") { + v.emplace_back(1); + v.emplace_back(2); + v.emplace_back(3); + }; + REQUIRE(v.size() == 0); + + std::size_t counter{0}; + BENCHMARK_ADVANCED("construct vector")(Catch::Benchmark::Chronometer meter) { + std::vector> storage(meter.runs()); + meter.measure([&](int i) { storage[i].construct("thing"); counter++; }); + }; + REQUIRE(counter == 0); +}