diff --git a/README.adoc b/README.adoc index e0e209b3e..4f9f96ded 100644 --- a/README.adoc +++ b/README.adoc @@ -130,6 +130,7 @@ The least we can do is to thank them and list some of their accomplishments here * https://github.com/eeverman[Eric Everman] added `@RestoreSystemProperties` and `@RestoreEnvironmentVariables` annotations to the https://junit-pioneer.org/docs/system-properties/[System Properties] and https://junit-pioneer.org/docs/environment-variables/[Environment Variables] extensions (#574 / #700) * https://github.com/meredrica[Florian Westreicher] contributed to the JSON argument source extension (#704 / #724) * https://github.com/IlyasYOY[Ilya Ilyinykh] found unused demo tests (#791) +* https://github.com/knutwannheden[Knut Wannheden] contributed the `withExceptions` attribute of the https://junit-pioneer.org/docs/expected-to-fail-tests/[`@ExpectedToFail` extension] (#769 / #774) * https://github.com/petrandreev[Pёtr Andreev] added back support for NULL values to `@CartesianTestExtension` (#764 / #765) ==== 2022 diff --git a/docs/expected-to-fail-tests.adoc b/docs/expected-to-fail-tests.adoc index ccceb4e80..6353cdd6c 100644 --- a/docs/expected-to-fail-tests.adoc +++ b/docs/expected-to-fail-tests.adoc @@ -44,6 +44,25 @@ A custom message can be provided, explaining why the tested code is not working include::{demo}[tag=expected_to_fail_message] ---- +=== Only Abort on Expected Exceptions + +A test that is `@ExpectedToFail` will change its behavior by starting to actually fail, once the code under test behaves correctly. +If the underlying failure changes, though, for example from an assertion error to an exception or from an `UnsupportedOperationException` of a formerly missing implementation to a runtime exception of buggy implementation, the `@ExpectedToFail`-test will keep passing. + +To better react to such changes, `@ExpectedToFail` has an attribute `withExceptions` that can be used to enumerate the exceptions which when thrown will result in an aborted (and thus passing) test. +Any other exception thrown by the code under test will result in a failing test. + +In the following example a test case for `productionFeature()` has been implemented. +While the test is fully implemented, the production code has been stubbed and throws an `UnsupportedOperationException` to indicate this. + +[source,java,indent=0] +---- +include::{demo}[tag=expected_to_fail_withexception] +---- + +Once `productionFeature()` is implemented, `@ExpectedToFail` will fail the test, as no `UnsupportedOperationException` is thrown anymore. +By using `withExceptions` you can thus prevent "masking" a faulty implementation (e.g. when a value other rather than `10` is returned) with an aborted test (which would be the result when no `withExceptions` is set). + == Thread-Safety This extension is safe to use during https://junit.org/junit5/docs/current/user-guide/#writing-tests-parallel-execution[parallel test execution]. diff --git a/src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java b/src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java index 0477375d7..d63059d7f 100644 --- a/src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java +++ b/src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java @@ -38,4 +38,17 @@ private int brokenMethod() { return 0; } + // tag::expected_to_fail_withexception[] + @Test + @ExpectedToFail(withExceptions = UnsupportedOperationException.class) + void testProductionFeature() { + int actual = productionFeature(); + assertThat(actual).isEqualTo(10); + } + + private int productionFeature() { + throw new UnsupportedOperationException("productionFeature() is not yet implemented"); + } + // end::expected_to_fail_withexception[] + } diff --git a/src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java b/src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java index bac53e04c..1ab0e2749 100644 --- a/src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java +++ b/src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java @@ -32,6 +32,13 @@ * This helps to avoid creating duplicate tests by accident and counteracts the accumulation * of disabled tests over time.

* + *

Further, the {@link #withExceptions()} attribute can be used to restrict the extension's behavior + * to specific exceptions. That is, only if the test method ends up throwing one of the specified exceptions + * will the test be aborted. This can, for example, be used when the production code temporarily throws + * an {@link UnsupportedOperationException} because some feature has not been implemented yet, but the + * test method is already implemented and should not fail on a failing assertion. + *

+ * *

The annotation can only be used on methods and as meta-annotation on other annotation types. * Similar to {@code @Disabled}, it has to be used in addition to a "testable" annotation, such * as {@link org.junit.jupiter.api.Test @Test}. Otherwise the annotation has no effect.

@@ -68,4 +75,11 @@ */ String value() default ""; + /** + * Specifies which exceptions are expected to be thrown and will cause the test to be aborted rather than fail. + * An empty array is considered a configuration error and will cause the test to fail. Instead, consider leaving + * the attribute set to the default value when any exception should cause the test to be aborted. + */ + Class[] withExceptions() default { Throwable.class }; + } diff --git a/src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java b/src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java index 0e13a5cf3..8f590d554 100644 --- a/src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java +++ b/src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java @@ -11,8 +11,10 @@ package org.junitpioneer.jupiter; import java.lang.reflect.Method; +import java.util.stream.Stream; import org.junit.jupiter.api.extension.Extension; +import org.junit.jupiter.api.extension.ExtensionConfigurationException; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.InvocationInterceptor; import org.junit.jupiter.api.extension.ReflectiveInvocationContext; @@ -30,6 +32,11 @@ public void interceptTestMethod(Invocation invocation, ReflectiveInvocatio private static void invokeAndInvertResult(Invocation invocation, ExtensionContext extensionContext) throws Throwable { + ExpectedToFail expectedToFail = getExpectedToFailAnnotation(extensionContext); + if (expectedToFail.withExceptions().length == 0) { + throw new ExtensionConfigurationException("@ExpectedToFail withExceptions must not be empty"); + } + try { invocation.proceed(); // at this point, the invocation succeeded, so we'd want to call `fail(...)`, @@ -41,7 +48,12 @@ private static void invokeAndInvertResult(Invocation invocation, Extension throw t; } - String message = getExpectedToFailAnnotation(extensionContext).value(); + if (Stream.of(expectedToFail.withExceptions()).noneMatch(clazz -> clazz.isInstance(t))) { + throw new AssertionFailedError( + "Test marked as temporarily 'expected to fail' failed with an unexpected exception", t); + } + + String message = expectedToFail.value(); if (message.isEmpty()) { message = "Test marked as temporarily 'expected to fail' failed as expected"; } diff --git a/src/test/java/org/junitpioneer/jupiter/ExpectedToFailExtensionTests.java b/src/test/java/org/junitpioneer/jupiter/ExpectedToFailExtensionTests.java index 3b0720d5a..5b271d0ca 100644 --- a/src/test/java/org/junitpioneer/jupiter/ExpectedToFailExtensionTests.java +++ b/src/test/java/org/junitpioneer/jupiter/ExpectedToFailExtensionTests.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionConfigurationException; import org.junitpioneer.testkit.ExecutionResults; import org.junitpioneer.testkit.PioneerTestKit; import org.opentest4j.TestAbortedException; @@ -100,6 +101,52 @@ void failsOnWorkingTest() { .hasMessage("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it"); } + @Test + void doesNotAbortOnTestThrowingExpectedException() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailTestCases.class, "withExceptionsExpected"); + assertThat(results) + .hasSingleStartedTest() + .whichAborted() + .withExceptionInstanceOf(TestAbortedException.class) + .hasMessage("Test marked as temporarily 'expected to fail' failed as expected") + .hasCauseInstanceOf(UnsupportedOperationException.class); + } + + @Test + void failsOnTestThrowingUnexpectedException() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailTestCases.class, "withExceptionsUnexpected"); + assertThat(results) + .hasSingleStartedTest() + .whichFailed() + .withExceptionInstanceOf(AssertionError.class) + .hasMessage("Test marked as temporarily 'expected to fail' failed with an unexpected exception") + .hasCauseInstanceOf(IllegalStateException.class); + } + + @Test + void failsOnWorkingTestWithExpectedException() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailTestCases.class, "withExceptionsWorking"); + assertThat(results) + .hasSingleStartedTest() + .whichFailed() + .withExceptionInstanceOf(AssertionError.class) + .hasMessage("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it"); + } + + @Test + void failsOnWorkingTestWithEmptyExpectedExceptions() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailTestCases.class, "withExceptionsEmpty"); + assertThat(results) + .hasSingleStartedTest() + .whichFailed() + .withExceptionInstanceOf(ExtensionConfigurationException.class) + .hasMessage("@ExpectedToFail withExceptions must not be empty"); + } + @Test void doesNotAbortOnBeforeEachTestFailure() { ExecutionResults results = PioneerTestKit @@ -218,6 +265,28 @@ void working() { // Does not cause failure or error } + @Test + @ExpectedToFail(withExceptions = { IllegalStateException.class, UnsupportedOperationException.class }) + void withExceptionsExpected() { + throw new UnsupportedOperationException(); + } + + @Test + @ExpectedToFail(withExceptions = UnsupportedOperationException.class) + void withExceptionsUnexpected() { + throw new IllegalStateException(); + } + + @Test + @ExpectedToFail(withExceptions = UnsupportedOperationException.class) + void withExceptionsWorking() { + } + + @Test + @ExpectedToFail(withExceptions = {}) + void withExceptionsEmpty() { + } + } /**