Add withExceptions attribute to @ExpectedToFail

README.md

@@ -121,6 +121,7 @@ The least we can do is to thank them and list some of their accomplishments here
 * [Eric Everman](https://github.com/eeverman) added `@RestoreSystemProperties` and `@RestoreEnvironmentVariables` annotations to the [System Properties](https://junit-pioneer.org/docs/system-properties/) and [Environment Variables](https://junit-pioneer.org/docs/environment-variables/) extensions (#574 / #700)
 * [Florian Westreicher](https://github.com/meredrica) contributed to the JSON argument source extension (#704 / #724)
 * [Pёtr Andreev](https://github.com/petrandreev) added back support for NULL values to `@CartesianTestExtension` (#764 / #765)
+* [Knut Wannheden](https://github.com/knutwannheden) contributed the `onExceptions` attribute of the [`@ExpectedToFail` extension](https://junit-pioneer.org/docs/expected-to-fail-tests/) (#769 / #774)
 
 #### 2022
 

docs/expected-to-fail-tests.adoc

@@ -44,6 +44,26 @@ A custom message can be provided, explaining why the tested code is not working
 include::{demo}[tag=expected_to_fail_message]
 ----
 
+=== Only Abort on Expected Exceptions
+
+In some cases it may be desirable to only have the test be aborted on certain expected exceptions.
+In the following example a test case for `productionFeature()` has been implemented.
+While the test is fully implemented, the production code has been stubbed and throws an `UnsupportedOperationException` to indicate this.
+
+[source,java,indent=0]
+----
+include::{demo}[tag=expected_to_fail_onexception]
+----
+
+Once `productionFeature()` is implemented, `@ExpectedToFail` will fail the test, as no `UnsupportedOperationException` is thrown anymore.
+By using `onExceptions` you can thus prevent "masking" a faulty implementation (e.g. when a value other rather than `10` is returned) with an aborted test (which would be the result when no `onExceptions` is set).
+
+[WARNING]
+====
+As stated in the introduction, this feature is _not_ intended as a mechanism to assert that a test method intentionally throws an exception.
+Instead, such test methods should use https://junit.org/junit5/docs/current/api/org.junit.jupiter.api/org/junit/jupiter/api/Assertions.html#assertThrows(java.lang.Class,org.junit.jupiter.api.function.Executable)[JUnit's `assertThrows`] or similar.
+====
+
 == Thread-Safety
 
 This extension is safe to use during https://junit.org/junit5/docs/current/user-guide/#writing-tests-parallel-execution[parallel test execution].

src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java

@@ -38,4 +38,17 @@ private int brokenMethod() {
 		return 0;
 	}
 
+	// tag::expected_to_fail_onexception[]
+	@Test
+	@ExpectedToFail(onExceptions = UnsupportedOperationException.class)
+	void testProductionFeature() {
+		int actual = productionFeature();
+		assertEquals(10, actual);
+	}
+
+	private int productionFeature() {
+		throw new UnsupportedOperationException("productionFeature() is not yet implemented");
+	}
+	// end::expected_to_fail_onexception[]
+
 }

src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java

@@ -33,6 +33,13 @@
  * of disabled tests over time.
  * </p>
  *
+ * <p>Further, the {@link #onExceptions()} attribute can be used to restrict the extension's behavior
+ * to specific exceptions. I.e. only if the test method end up throwing one of the specified exceptions
+ * will the test be aborted. This can, for example, be used when the production code temporarily throws
+ * an {@link UnsupportedOperationException} because some feature has not been implemented yet, but the
+ * test method is already implemented and should not fail on a failing assertion.
+ * </p>
+ *
  * <p>The annotation can only be used on methods and as meta-annotation on other annotation types.
  * Similar to {@code @Disabled}, it has to be used in addition to a "testable" annotation, such
  * as {@link org.junit.jupiter.api.Test @Test}. Otherwise the annotation has no effect.
@@ -41,8 +48,7 @@
  * <p><b>Important:</b> This annotation is <b>not</b> intended as a way to mark test methods
  * which intentionally cause exceptions. Such test methods should use
  * {@link org.junit.jupiter.api.Assertions#assertThrows(Class, org.junit.jupiter.api.function.Executable) assertThrows}
- * or similar means to explicitly test for a specific exception class being thrown by a
- * specific action.
+ * or similar means to explicitly test for a specific exception class being thrown by a specific action.
  * </p>
  *
  * <p>For more details and examples, see
@@ -72,4 +78,9 @@
 	 */
 	String value() default "";
 
+	/**
+	 * Specifies which exceptions are expected to be thrown and will cause the test to be aborted rather than fail.
+	 */
+	Class<? extends Throwable>[] onExceptions() default { Throwable.class };
+
 }

src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java

@@ -13,6 +13,7 @@
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.lang.reflect.Method;
+import java.util.stream.Stream;
 
 import org.junit.jupiter.api.extension.Extension;
 import org.junit.jupiter.api.extension.ExtensionContext;
@@ -42,7 +43,12 @@ private static void invokeAndInvertResult(Invocation<Void> invocation, Extension
 				throw t;
 			}
 
-			String message = getExpectedToFailAnnotation(extensionContext).value();
+			ExpectedToFail expectedToFail = getExpectedToFailAnnotation(extensionContext);
+			if (Stream.of(expectedToFail.onExceptions()).noneMatch(clazz -> clazz.isInstance(t))) {
+				fail("Test marked as 'expected to fail' failed with an unexpected exception", t);
+			}
+
+			String message = expectedToFail.value();
 			if (message.isEmpty()) {
 				message = "Test marked as temporarily 'expected to fail' failed as expected";
 			}

src/test/java/org/junitpioneer/jupiter/ExpectedToFailExtensionTests.java

@@ -101,6 +101,29 @@ void failsOnWorkingTest() {
 				.hasMessage("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it");
 	}
 
+	@Test
+	void doesNotAbortOnTestThrowingExpectedException() {
+		ExecutionResults results = PioneerTestKit.executeTestMethod(ExpectedToFailTestCases.class, "expectedException");
+		assertThat(results)
+				.hasSingleStartedTest()
+				.whichAborted()
+				.withExceptionInstanceOf(TestAbortedException.class)
+				.hasMessage("Test marked as temporarily 'expected to fail' failed as expected")
+				.hasCauseInstanceOf(UnsupportedOperationException.class);
+	}
+
+	@Test
+	void failsOnTestThrowingUnexpectedException() {
+		ExecutionResults results = PioneerTestKit
+				.executeTestMethod(ExpectedToFailTestCases.class, "unexpectedException");
+		assertThat(results)
+				.hasSingleStartedTest()
+				.whichFailed()
+				.withExceptionInstanceOf(AssertionFailedError.class)
+				.hasMessage("Test marked as 'expected to fail' failed with an unexpected exception")
+				.hasCauseInstanceOf(AssertionFailedError.class);
+	}
+
 	@Test
 	void doesNotAbortOnBeforeEachTestFailure() {
 		ExecutionResults results = PioneerTestKit
@@ -219,6 +242,18 @@ void working() {
 			// Does not cause failure or error
 		}
 
+		@Test
+		@ExpectedToFail(onExceptions = { IllegalStateException.class, UnsupportedOperationException.class })
+		void expectedException() {
+			throw new UnsupportedOperationException();
+		}
+
+		@Test
+		@ExpectedToFail(onExceptions = UnsupportedOperationException.class)
+		void unexpectedException() {
+			fail();
+		}
+
 	}
 
 	/**