Add withExceptions attribute to @ExpectedToFail

README.adoc

@@ -125,6 +125,7 @@ The least we can do is to thank them and list some of their accomplishments here
 * https://github.com/eeverman[Eric Everman] added `@RestoreSystemProperties` and `@RestoreEnvironmentVariables` annotations to the https://junit-pioneer.org/docs/system-properties/[System Properties] and https://junit-pioneer.org/docs/environment-variables/[Environment Variables] extensions (#574 / #700)
 * https://github.com/meredrica[Florian Westreicher] contributed to the JSON argument source extension (#704 / #724)
 * https://github.com/IlyasYOY[Ilya Ilyinykh] found unused demo tests (#791)
+* https://github.com/knutwannheden[Knut Wannheden] contributed the `withExceptions` attribute of the https://junit-pioneer.org/docs/expected-to-fail-tests/[`@ExpectedToFail` extension] (#769 / #774)
 * https://github.com/petrandreev[Pёtr Andreev] added back support for NULL values to `@CartesianTestExtension` (#764 / #765)
 
 ==== 2022

docs/expected-to-fail-tests.adoc

@@ -44,6 +44,25 @@ A custom message can be provided, explaining why the tested code is not working
 include::{demo}[tag=expected_to_fail_message]
 ----
 
+=== Only Abort on Expected Exceptions
+
+A test that is `@ExpectedToFail` will change its behavior by starting to actually fail, once the code under test behaves correctly.
+If the underlying failure changes, though, for example from an assertion error to an exception or from an `UnsupportedOperationException` of a formerly missing implementation to a runtime exception of buggy implementation, the `@ExpectedToFail`-test will keep passing.
+
+To better react to such changes, `@ExpectedToFail` has an attribute `withExceptions` that can be used to enumerate the exceptions which when thrown will result in an aborted (and thus passing) test.
+Any other exception thrown by the code under test will result in a failing test.
+
+In the following example a test case for `productionFeature()` has been implemented.
+While the test is fully implemented, the production code has been stubbed and throws an `UnsupportedOperationException` to indicate this.
+
+[source,java,indent=0]
+----
+include::{demo}[tag=expected_to_fail_withexception]
+----
+
+Once `productionFeature()` is implemented, `@ExpectedToFail` will fail the test, as no `UnsupportedOperationException` is thrown anymore.
+By using `withExceptions` you can thus prevent "masking" a faulty implementation (e.g. when a value other rather than `10` is returned) with an aborted test (which would be the result when no `withExceptions` is set).
+
 == Thread-Safety
 
 This extension is safe to use during https://junit.org/junit5/docs/current/user-guide/#writing-tests-parallel-execution[parallel test execution].

src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java

@@ -38,4 +38,17 @@ private int brokenMethod() {
 		return 0;
 	}
 
+	// tag::expected_to_fail_withexception[]
+	@Test
+	@ExpectedToFail(withExceptions = UnsupportedOperationException.class)
+	void testProductionFeature() {
+		int actual = productionFeature();
+		assertEquals(10, actual);
+	}
+
+	private int productionFeature() {
+		throw new UnsupportedOperationException("productionFeature() is not yet implemented");
+	}
+	// end::expected_to_fail_withexception[]
+
 }

src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java

@@ -32,6 +32,13 @@
  * This helps to avoid creating duplicate tests by accident and counteracts the accumulation
  * of disabled tests over time.</p>
  *
+ * <p>Further, the {@link #withExceptions()} attribute can be used to restrict the extension's behavior
+ * to specific exceptions. That is, only if the test method ends up throwing one of the specified exceptions
+ * will the test be aborted. This can, for example, be used when the production code temporarily throws
+ * an {@link UnsupportedOperationException} because some feature has not been implemented yet, but the
+ * test method is already implemented and should not fail on a failing assertion.
+ * </p>
+ *
  * <p>The annotation can only be used on methods and as meta-annotation on other annotation types.
  * Similar to {@code @Disabled}, it has to be used in addition to a "testable" annotation, such
  * as {@link org.junit.jupiter.api.Test @Test}. Otherwise the annotation has no effect.</p>
@@ -68,4 +75,11 @@
 	 */
 	String value() default "";
 
+	/**
+	 * Specifies which exceptions are expected to be thrown and will cause the test to be aborted rather than fail.
+	 * An empty array is considered a configuration error and will cause the test to fail. Instead, consider leaving
+	 * the attribute set to the default value when any exception should cause the test to be aborted.
+	 */
+	Class<? extends Throwable>[] withExceptions() default { Throwable.class };
+
 }

src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java

@@ -13,8 +13,10 @@
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.lang.reflect.Method;
+import java.util.stream.Stream;
 
 import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionConfigurationException;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.InvocationInterceptor;
 import org.junit.jupiter.api.extension.ReflectiveInvocationContext;
@@ -31,6 +33,11 @@ public void interceptTestMethod(Invocation<Void> invocation, ReflectiveInvocatio
 
 	private static void invokeAndInvertResult(Invocation<Void> invocation, ExtensionContext extensionContext)
 			throws Throwable {
+		ExpectedToFail expectedToFail = getExpectedToFailAnnotation(extensionContext);
+		if (expectedToFail.withExceptions().length == 0) {
+			throw new ExtensionConfigurationException("@ExpectedToFail withExceptions must not be empty");
+		}
+
 		try {
 			invocation.proceed();
 			// at this point, the invocation succeeded, so we'd want to call `fail(...)`,
@@ -42,7 +49,11 @@ private static void invokeAndInvertResult(Invocation<Void> invocation, Extension
 				throw t;
 			}
 
-			String message = getExpectedToFailAnnotation(extensionContext).value();
+			if (Stream.of(expectedToFail.withExceptions()).noneMatch(clazz -> clazz.isInstance(t))) {
+				fail("Test marked as temporarily 'expected to fail' failed with an unexpected exception", t);
+			}
+
+			String message = expectedToFail.value();
 			if (message.isEmpty()) {
 				message = "Test marked as temporarily 'expected to fail' failed as expected";
 			}

src/test/java/org/junitpioneer/jupiter/ExpectedToFailExtensionTests.java

@@ -26,6 +26,7 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtensionConfigurationException;
 import org.junitpioneer.testkit.ExecutionResults;
 import org.junitpioneer.testkit.PioneerTestKit;
 import org.opentest4j.AssertionFailedError;
@@ -101,6 +102,52 @@ void failsOnWorkingTest() {
 				.hasMessage("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it");
 	}
 
+	@Test
+	void doesNotAbortOnTestThrowingExpectedException() {
+		ExecutionResults results = PioneerTestKit
+				.executeTestMethod(ExpectedToFailTestCases.class, "withExceptionsExpected");
+		assertThat(results)
+				.hasSingleStartedTest()
+				.whichAborted()
+				.withExceptionInstanceOf(TestAbortedException.class)
+				.hasMessage("Test marked as temporarily 'expected to fail' failed as expected")
+				.hasCauseInstanceOf(UnsupportedOperationException.class);
+	}
+
+	@Test
+	void failsOnTestThrowingUnexpectedException() {
+		ExecutionResults results = PioneerTestKit
+				.executeTestMethod(ExpectedToFailTestCases.class, "withExceptionsUnexpected");
+		assertThat(results)
+				.hasSingleStartedTest()
+				.whichFailed()
+				.withExceptionInstanceOf(AssertionFailedError.class)
+				.hasMessage("Test marked as temporarily 'expected to fail' failed with an unexpected exception")
+				.hasCauseInstanceOf(AssertionFailedError.class);
+	}
+
+	@Test
+	void failsOnWorkingTestWithExpectedException() {
+		ExecutionResults results = PioneerTestKit
+				.executeTestMethod(ExpectedToFailTestCases.class, "withExceptionsWorking");
+		assertThat(results)
+				.hasSingleStartedTest()
+				.whichFailed()
+				.withExceptionInstanceOf(AssertionFailedError.class)
+				.hasMessage("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it");
+	}
+
+	@Test
+	void failsOnWorkingTestWithEmptyExpectedExceptions() {
+		ExecutionResults results = PioneerTestKit
+				.executeTestMethod(ExpectedToFailTestCases.class, "withExceptionsEmpty");
+		assertThat(results)
+				.hasSingleStartedTest()
+				.whichFailed()
+				.withExceptionInstanceOf(ExtensionConfigurationException.class)
+				.hasMessage("@ExpectedToFail withExceptions must not be empty");
+	}
+
 	@Test
 	void doesNotAbortOnBeforeEachTestFailure() {
 		ExecutionResults results = PioneerTestKit
@@ -219,6 +266,28 @@ void working() {
 			// Does not cause failure or error
 		}
 
+		@Test
+		@ExpectedToFail(withExceptions = { IllegalStateException.class, UnsupportedOperationException.class })
+		void withExceptionsExpected() {
+			throw new UnsupportedOperationException();
+		}
+
+		@Test
+		@ExpectedToFail(withExceptions = UnsupportedOperationException.class)
+		void withExceptionsUnexpected() {
+			fail();
+		}
+
+		@Test
+		@ExpectedToFail(withExceptions = UnsupportedOperationException.class)
+		void withExceptionsWorking() {
+		}
+
+		@Test
+		@ExpectedToFail(withExceptions = {})
+		void withExceptionsEmpty() {
+		}
+
 	}
 
 	/**