From 6309b3cd903b6feb88563a9a120e29777e4cd334 Mon Sep 17 00:00:00 2001 From: Philip Salzmann Date: Mon, 9 Jan 2023 15:48:19 +0100 Subject: [PATCH] Add docs for SKIP macro, deprecate IEventListener::skipTest --- docs/Readme.md | 1 + docs/command-line.md | 8 +-- docs/deprecations.md | 9 +++ docs/skipping.md | 72 +++++++++++++++++++ .../interfaces/catch_interfaces_reporter.hpp | 7 +- 5 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 docs/skipping.md diff --git a/docs/Readme.md b/docs/Readme.md index 52eb64a50f..126ee1be9b 100644 --- a/docs/Readme.md +++ b/docs/Readme.md @@ -11,6 +11,7 @@ Once you're up and running consider the following reference material. * [Logging macros](logging.md#top) * [Test cases and sections](test-cases-and-sections.md#top) * [Test fixtures](test-fixtures.md#top) +* [Skipping tests at runtime](skipping.md#top) * [Reporters (output customization)](reporters.md#top) * [Event Listeners](event-listeners.md#top) * [Data Generators (value parameterized tests)](generators.md#top) diff --git a/docs/command-line.md b/docs/command-line.md index 6fc99f59e7..e7a448b165 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -561,10 +561,10 @@ processes, as is done with the [Bazel test sharding](https://docs.bazel.build/ve > Introduced in Catch2 3.0.1. -By default, Catch2 test binaries return non-0 exit code if no tests were -run, e.g. if the binary was compiled with no tests, or the provided test -spec matched no tests. This flag overrides that, so a test run with no -tests still returns 0. +By default, Catch2 test binaries return non-0 exit code if no tests were run, +e.g. if the binary was compiled with no tests, the provided test spec matched no +tests, or all tests [were skipped at runtime](skipping.md#top). This flag +overrides that, so a test run with no tests still returns 0. ## Output verbosity ``` diff --git a/docs/deprecations.md b/docs/deprecations.md index 2c9bf5517b..5926f2c541 100644 --- a/docs/deprecations.md +++ b/docs/deprecations.md @@ -26,6 +26,15 @@ to accurately probe the environment for this information so the flag where it will export `BAZEL_TEST=1` for purposes like the above. Catch2 will now instead inspect the environment instead of relying on build configuration. +### `IEventLister::skipTest( TestCaseInfo const& testInfo )` + +This event (including implementations in derived classes such as `ReporterBase`) +is deprecated and will be removed in the next major release. It is currently +invoked for all test cases that are not going to be executed due to the test run +being aborted (when using `--abort` or `--abortx`). It is however +**NOT** invoked for test cases that are [explicitly skipped using the `SKIP` +macro](skipping.md#top). + --- [Home](Readme.md#top) diff --git a/docs/skipping.md b/docs/skipping.md new file mode 100644 index 0000000000..edd2a1a1ae --- /dev/null +++ b/docs/skipping.md @@ -0,0 +1,72 @@ + +# Skipping Test Cases at Runtime + +> [Introduced](https://github.com/catchorg/Catch2/pull/2360) in Catch2 X.Y.Z. + +In some situations it may not be possible to meaningfully execute a test case, for example when the system under test is missing certain hardware capabilities. +If the required conditions can only be determined at runtime, it often doesn't make sense to consider such a test case as either passed or failed, because it simply can not run at all. +To properly express such scenarios, Catch2 allows to explicitly _skip_ test cases, using the `SKIP` macro: + +**SKIP(** _message expression_ **)** + +Example usage: + +```c++ +TEST_CASE("copy files between drives") { + if(getNumberOfHardDrives() < 2) { + SKIP("at least two hard drives required"); + } + // ... +} +``` + +This test case is then reported as _skipped_ instead of _passed_ or _failed_. + +The `SKIP` macro behaves similarly to an explicit [`FAIL`](logging.md#top), in that it is the last expression that will be executed: + +```c++ +TEST_CASE("my test") { + printf("foo"); + SKIP(); + printf("bar"); // not printed +} +``` + +However a failed assertion _before_ a `SKIP` still causes the entire test case to fail: + +```c++ +TEST_CASE("failing test") { + CHECK(1 == 2); + SKIP(); +} +``` + +## Interaction with Sections and Generators + +Sections, nested sections as well as test cases with [generators](generators.md#top) can all be individually skipped, with the rest executing as usual: + +```c++ +TEST_CASE("complex test case") { + int value = GENERATE(2, 4, 6); + SECTION("a") { + SECTION("a1") { CHECK(value < 8); } + SECTION("a2") { + if (value == 4) { + SKIP(); + } + CHECK(value % 2 == 0); + } + } +} +``` + +This test case will report 5 passing assertions; one for each of the three values in section `a1`, as well as one for each in `a2`, except for when `value == 4`. + +Note that as soon as one section is skipped, the entire test case will be reported as _skipped_ (unless there is a failing assertion, in which case it will be reported as _failed_ instead). + +If all test cases in a run are skipped, Catch2 returns a non-zero exit code by default. +This can be overridden using the [--allow-running-no-tests](command-line.md#no-tests-override) flag. + +--- + +[Home](Readme.md#top) diff --git a/src/catch2/interfaces/catch_interfaces_reporter.hpp b/src/catch2/interfaces/catch_interfaces_reporter.hpp index 5f28636332..da0112e3f8 100644 --- a/src/catch2/interfaces/catch_interfaces_reporter.hpp +++ b/src/catch2/interfaces/catch_interfaces_reporter.hpp @@ -242,7 +242,12 @@ namespace Catch { */ virtual void testRunEnded( TestRunStats const& testRunStats ) = 0; - //! Called with test cases that are skipped due to the test run aborting + /** + * Called with test cases that are skipped due to the test run aborting. + * NOT called for test cases that are explicitly skipped using the `SKIP` macro. + * + * Deprecated - will be removed in the next major release. + */ virtual void skipTest( TestCaseInfo const& testInfo ) = 0; //! Called if a fatal error (signal/structured exception) occured