diff --git a/README.md b/README.md index 0a2148af8..195bf66b3 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ The least we can do is to thank them and list some of their accomplishments here #### 2022 * [Filip Hrisafov](https://github.com/filiphr) contributed the [JSON Argument Source](https://junit-pioneer.org/docs/json-argument-source/) support (#101 / #492) +* [Marcono1234](https://github.com/Marcono1234) contributed the [`@ExpectedToFail` extension](https://junit-pioneer.org/docs/expected-to-fail-tests/) (#551 / #676) * [Mathieu Fortin](https://github.com/mathieufortin01) contributed the `suspendForMs` attribute in [retrying tests](https://junit-pioneer.org/docs/retrying-test/) (#407 / #604) * [Pankaj Kumar](https://github.com/p1729) contributed towards improving GitHub actions (#587 / #611) * [Rob Spoor](https://github.com/robtimus) enabled non-static factory methods for `@CartesianTest.MethodFactory` (#628) diff --git a/docs/docs-nav.yml b/docs/docs-nav.yml index 1492235fb..3a1e9e68b 100644 --- a/docs/docs-nav.yml +++ b/docs/docs-nav.yml @@ -16,6 +16,8 @@ url: /docs/disable-if-test-fails/ - title: "Disable Parameterized Tests" url: /docs/disable-parameterized-tests/ + - title: "Expected-to-Fail Tests" + url: /docs/expected-to-fail-tests/ - title: "Issue information" url: /docs/issue/ - title: "JSON Argument Source" diff --git a/docs/expected-to-fail-tests.adoc b/docs/expected-to-fail-tests.adoc new file mode 100644 index 000000000..cb4562387 --- /dev/null +++ b/docs/expected-to-fail-tests.adoc @@ -0,0 +1,46 @@ +:page-title: Expected to Fail Tests +:page-description: Extends JUnit Jupiter with `@ExpectedToFail`, which marks a test method as temporarily 'expected to fail' +:xp-demo-dir: ../src/demo/java +:demo: {xp-demo-dir}/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java + +Often tests fail due to a bug in the tested application or in dependencies. +Traditionally such a test method would be annotated with JUnit's `@Disabled`. +However, this has disadvantages when the bug that causes the test failure is fixed: + +* the developer might not notice the existing test method and create a new one +* the existing test method might not be noticed and remains disabled for a long time after the bug has been fixed, adding no value for the project + +`@ExpectedToFail` solves these issues. +Unlike `@Disabled` it still executes the annotated test method but aborts the test if a test failure or error occurs. +However, if the test is executed successfully it will cause a test failure because the test _is working_. +This lets the developer know that they have fixed the bug (possibly by accident) and that they can now remove the `@ExpectedToFail` annotation from the test method. + +The annotation can only be used on methods and as meta-annotation on other annotation types. +Similar to `@Disabled`, it has to be used in addition to a "testable" annotation, such as `@Test`. +Otherwise the annotation has no effect. + +IMPORTANT: This annotation is *not* intended as a way to mark test methods which intentionally cause exceptions. +Such test methods should use https://junit.org/junit5/docs/current/api/org.junit.jupiter.api/org/junit/jupiter/api/Assertions.html#assertThrows(java.lang.Class,org.junit.jupiter.api.function.Executable)[JUnit's `assertThrows`] or similar means to explicitly test for a specific exception class being thrown by a specific action. + +== Basic Use + +The test is aborted because the tested method `brokenMethod()` returns an incorrect result. + +[source,java,indent=0] +---- +include::{demo}[tag=expected_to_fail] +---- + +An aborted test is no failure and so the test suite passes (if all other tests pass, of course). +Should `brokenMethod()` start returning the correct value, the test invocation passes, but `@ExpectedToFail` marks the test as failed to draw attention to that change in behavior. + +A custom message can be provided, explaining why the tested code is not working as intended at the moment. + +[source,java,indent=0] +---- +include::{demo}[tag=expected_to_fail_message] +---- + +== Thread-Safety + +This extension is safe to use during https://junit.org/junit5/docs/current/user-guide/#writing-tests-parallel-execution[parallel test execution]. diff --git a/src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java b/src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java new file mode 100644 index 000000000..78676c701 --- /dev/null +++ b/src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java @@ -0,0 +1,41 @@ +/* + * Copyright 2016-2022 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * http://www.eclipse.org/legal/epl-v20.html + */ + +package org.junitpioneer.jupiter; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +public class ExpectedToFailExtensionDemo { + + // tag::expected_to_fail[] + @Test + @ExpectedToFail + void test() { + int actual = brokenMethod(); + assertEquals(10, actual); + } + // end::expected_to_fail[] + + // tag::expected_to_fail_message[] + @Test + @ExpectedToFail("Implementation bug in brokenMethod()") + void doSomething() { + int actual = brokenMethod(); + assertEquals(10, actual); + } + // end::expected_to_fail_message[] + + private int brokenMethod() { + return 0; + } + +} diff --git a/src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java b/src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java new file mode 100644 index 000000000..7df0f60ff --- /dev/null +++ b/src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java @@ -0,0 +1,72 @@ +/* + * Copyright 2016-2022 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * http://www.eclipse.org/legal/epl-v20.html + */ + +package org.junitpioneer.jupiter; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * {@code @ExpectedToFail} is a JUnit Jupiter extension to mark test methods as temporarily + * 'expected to fail'. Such test methods will still be executed but when they result in a test + * failure or error the test will be aborted. However, if the test method unexpectedly executes + * successfully, it is marked as failure to let the developer know that the test is now + * successful and that the {@code @ExpectedToFail} annotation can be removed. + * + *

The big difference compared to JUnit's {@link org.junit.jupiter.api.Disabled @Disabled} + * annotation is that the developer is informed as soon as a test is successful again. + * This helps to avoid creating duplicate tests by accident and counteracts the accumulation + * of disabled tests over time. + * + *

The annotation can only be used on methods and as meta-annotation on other annotation types. + * Similar to {@code @Disabled}, it has to be used in addition to a "testable" annotation, such + * as {@link org.junit.jupiter.api.Test @Test}. Otherwise the annotation has no effect. + * + *

Important: This annotation is not intended as a way to mark test methods + * which intentionally cause exceptions. Such test methods should use + * {@link org.junit.jupiter.api.Assertions#assertThrows(Class, org.junit.jupiter.api.function.Executable) assertThrows} + * or similar means to explicitly test for a specific exception class being thrown by a + * specific action. + * + *

For more details and examples, see + * the documentation on @ExpectedToFail. + *

+ * + * @since 1.8.0 + * @see org.junit.jupiter.api.Disabled + */ +@Documented +@Retention(RUNTIME) +/* + * Only supports METHOD and ANNOTATION_TYPE as targets but not test classes because there + * it is not clear what the 'correct' behavior would be when only a few test methods + * execute successfully. Would the developer then have to remove the @ExpectedToFail annotation + * from the test class and annotate methods individually? + */ +@Target({ METHOD, ANNOTATION_TYPE }) +@ExtendWith(ExpectedToFailExtension.class) +public @interface ExpectedToFail { + + /** + * Defines the message to show when a test is aborted because it is failing. + * This can be used for example to briefly explain why the tested code is not working + * as intended at the moment. + * An empty string (the default) causes a generic default message to be used. + */ + String value() default ""; + +} diff --git a/src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java b/src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java new file mode 100644 index 000000000..ed1930661 --- /dev/null +++ b/src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java @@ -0,0 +1,76 @@ +/* + * Copyright 2016-2022 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * http://www.eclipse.org/legal/epl-v20.html + */ + +package org.junitpioneer.jupiter; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.lang.reflect.Method; + +import org.junit.jupiter.api.extension.Extension; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.InvocationInterceptor; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; +import org.junit.platform.commons.support.AnnotationSupport; +import org.opentest4j.TestAbortedException; + +class ExpectedToFailExtension implements Extension, InvocationInterceptor { + + @Override + public void interceptTestMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext) throws Throwable { + invokeAndInvertResult(invocation, extensionContext); + } + + private static T invokeAndInvertResult(Invocation invocation, ExtensionContext extensionContext) + throws Throwable { + try { + invocation.proceed(); + // at this point, the invocation succeeded, so we'd want to call `fail(...)`, + // but that would get handled by the following `catch` and so it's easier + // to instead fall through to a `fail(...)` after the `catch` block + } + catch (Throwable t) { + if (shouldPreserveException(t)) { + throw t; + } + + String message = getExpectedToFailAnnotation(extensionContext).value(); + if (message.isEmpty()) { + message = "Test marked as temporarily 'expected to fail' failed as expected"; + } + + throw new TestAbortedException(message, t); + } + + return fail("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it"); + } + + /** + * Returns whether the exception should be preserved and reported as is instead + * of considering it an 'expected to fail' exception. + * + *

This method is used for exceptions that abort test execution and should + * have higher precedence than aborted exceptions thrown by this extension. + */ + private static boolean shouldPreserveException(Throwable t) { + // Note: Ideally would use the same logic JUnit uses to determine if exception is aborting + // execution, see its class OpenTest4JAndJUnit4AwareThrowableCollector + return TestAbortedException.class.isInstance(t); + } + + private static ExpectedToFail getExpectedToFailAnnotation(ExtensionContext context) { + return AnnotationSupport + .findAnnotation(context.getRequiredTestMethod(), ExpectedToFail.class) + .orElseThrow(() -> new IllegalStateException("@ExpectedToFail is missing.")); + + } + +} diff --git a/src/main/java/org/junitpioneer/jupiter/package-info.java b/src/main/java/org/junitpioneer/jupiter/package-info.java index 614c84fd7..b9571e46e 100644 --- a/src/main/java/org/junitpioneer/jupiter/package-info.java +++ b/src/main/java/org/junitpioneer/jupiter/package-info.java @@ -8,6 +8,7 @@ *

  • {@link org.junitpioneer.jupiter.DefaultLocale} and {@link org.junitpioneer.jupiter.DefaultTimeZone}
    • *
  • {@link org.junitpioneer.jupiter.DisabledUntil}
    • *
  • {@link org.junitpioneer.jupiter.DisableIfTestFails}
    • + *
  • {@link org.junitpioneer.jupiter.ExpectedToFail}
    • *
  • {@link org.junitpioneer.jupiter.ReportEntry}
    • *
  • {@link org.junitpioneer.jupiter.RetryingTest}
    • *
  • {@link org.junitpioneer.jupiter.StdIo}
    • diff --git a/src/test/java/org/junitpioneer/jupiter/ExpectedToFailExtensionTests.java b/src/test/java/org/junitpioneer/jupiter/ExpectedToFailExtensionTests.java new file mode 100644 index 000000000..d3dba337c --- /dev/null +++ b/src/test/java/org/junitpioneer/jupiter/ExpectedToFailExtensionTests.java @@ -0,0 +1,328 @@ +/* + * Copyright 2016-2022 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * http://www.eclipse.org/legal/epl-v20.html + */ + +package org.junitpioneer.jupiter; + +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.RetentionPolicy.RUNTIME; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junitpioneer.testkit.assertion.PioneerAssert.assertThat; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import org.assertj.core.api.Condition; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junitpioneer.testkit.ExecutionResults; +import org.junitpioneer.testkit.PioneerTestKit; +import org.opentest4j.AssertionFailedError; +import org.opentest4j.TestAbortedException; + +@DisplayName("ExpectedToFail extension") +public class ExpectedToFailExtensionTests { + + @Test + void abortsOnTestFailure() { + ExecutionResults results = PioneerTestKit.executeTestMethod(ExpectedToFailTestCases.class, "failure"); + assertThat(results) + .hasSingleStartedTest() + .whichAborted() + .withExceptionInstanceOf(TestAbortedException.class) + .hasMessage("Test marked as temporarily 'expected to fail' failed as expected") + .hasCause(new AssertionFailedError("failed")); + } + + @Test + void abortsOnTestFailureWithMetaAnnotation() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailTestCases.class, "metaAnnotationFailure"); + assertThat(results) + .hasSingleStartedTest() + .whichAborted() + .withExceptionInstanceOf(TestAbortedException.class) + .hasMessage("Test marked as temporarily 'expected to fail' failed as expected") + .hasCause(new AssertionFailedError("failed")); + } + + @Test + void abortsOnTestFailureWithMessage() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailTestCases.class, "failureWithMessage"); + assertThat(results) + .hasSingleStartedTest() + .whichAborted() + .withExceptionInstanceOf(TestAbortedException.class) + .hasMessage("Custom message") + .hasCause(new AssertionFailedError("failed")); + } + + @Test + void abortsOnException() { + ExecutionResults results = PioneerTestKit.executeTestMethod(ExpectedToFailTestCases.class, "exception"); + assertThat(results) + .hasSingleStartedTest() + .whichAborted() + .withExceptionInstanceOf(TestAbortedException.class) + .hasMessage("Test marked as temporarily 'expected to fail' failed as expected") + .hasCause(new RuntimeException("test")); + } + + @Test + void preservesTestAbort() { + ExecutionResults results = PioneerTestKit.executeTestMethod(ExpectedToFailTestCases.class, "aborted"); + assertThat(results) + .hasSingleStartedTest() + .whichAborted() + .withExceptionInstanceOf(TestAbortedException.class) + // Ignore message prefix generated by JUnit + .hasMessageEndingWith("custom assumption message"); + } + + @Test + void failsOnWorkingTest() { + ExecutionResults results = PioneerTestKit.executeTestMethod(ExpectedToFailTestCases.class, "working"); + assertThat(results) + .hasSingleStartedTest() + .whichFailed() + .withExceptionInstanceOf(AssertionError.class) + .hasMessage("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it"); + } + + @Test + void doesNotAbortOnBeforeEachTestFailure() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailFailureBeforeEachTestCases.class, "test"); + assertThat(results) + .hasSingleFailedTest() + .withExceptionInstanceOf(AssertionError.class) + .hasMessage("before each"); + } + + @Test + void doesNotAbortOnAfterEachTestFailure() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailFailureAfterEachTestCases.class, "test"); + assertThat(results) + .hasSingleFailedTest() + .withExceptionInstanceOf(AssertionError.class) + .hasMessage("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it") + // Note: This check for suppressed exception actually tests JUnit platform behavior + .hasSuppressedException(new AssertionFailedError("after each")); + } + + @Test + void doesNotAbortOnAfterEachTestFailureAfterTestAbort() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailAbortThenFailureAfterEachTestCases.class, "test"); + assertThat(results) + .hasSingleFailedTest() + .withExceptionInstanceOf(AssertionError.class) + .hasMessage("after each"); + } + + @Test + void afterEachAbortAfterTestFailure() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailFailureThenAbortAfterEachTestCases.class, "test"); + assertThat(results) + .hasSingleStartedTest() + .whichAborted() + .withExceptionInstanceOf(TestAbortedException.class) + .hasMessage("Test marked as temporarily 'expected to fail' failed as expected") + .hasCause(new AssertionFailedError("failed")) + // Note: This check for suppressed exception actually tests JUnit platform behavior + .has(new Condition<>((Throwable throwable) -> { + Throwable[] suppressed = throwable.getSuppressed(); + return suppressed.length == 1 && suppressed[0] instanceof TestAbortedException + // Ignore message prefix generated by JUnit + && suppressed[0].getMessage().endsWith("custom assumption message"); + }, "suppressed JUnit abort exception")); + } + + @Test + void doesNotAbortOnBeforeAllTestFailure() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailFailureBeforeAllTestCases.class, "test"); + assertThat(results).hasNumberOfStartedTests(0); + assertThat(results) + .hasSingleFailedContainer() + .withExceptionInstanceOf(AssertionError.class) + .hasMessage("before all"); + } + + @Test + void doesNotAbortOnAfterAllTestFailure() { + ExecutionResults results = PioneerTestKit + .executeTestMethod(ExpectedToFailFailureAfterAllTestCases.class, "test"); + assertThat(results) + .hasSingleStartedTest() + .whichFailed() + .withExceptionInstanceOf(AssertionError.class) + .hasMessage("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it"); + } + + static class ExpectedToFailTestCases { + + @ExpectedToFail + @Retention(RUNTIME) + @Target(METHOD) + @interface ExpectedToFailMetaAnnotation { + } + + @Test + @ExpectedToFail + void failure() { + fail("failed"); + } + + @Test + @ExpectedToFailMetaAnnotation + void metaAnnotationFailure() { + fail("failed"); + } + + @Test + @ExpectedToFail("Custom message") + void failureWithMessage() { + fail("failed"); + } + + @Test + @ExpectedToFail + void exception() { + throw new RuntimeException("test"); + } + + @Test + @ExpectedToFail + void aborted() { + // Assumption should have higher precedence than @ExpectedToFail + Assumptions.assumeTrue(false, "custom assumption message"); + } + + @Test + @ExpectedToFail + void working() { + // Does not cause failure or error + } + + } + + /** + * {@link BeforeEach} should not be considered by {@link ExpectedToFail} because it + * is not specific to the annotated test method. + */ + static class ExpectedToFailFailureBeforeEachTestCases { + + @BeforeEach + void beforeEach() { + fail("before each"); + } + + @Test + @ExpectedToFail + void test() { + } + + } + + /** + * {@link AfterEach} should be considered by {@link ExpectedToFail} because it + * might fail due to changes made to the test instance by the test method. + */ + static class ExpectedToFailFailureAfterEachTestCases { + + @AfterEach + void afterEach() { + fail("after each"); + } + + @Test + @ExpectedToFail + void test() { + } + + } + + /** + * Abort in test method followed by failure in {@link AfterEach} method should + * be treated as expected failure. + */ + static class ExpectedToFailAbortThenFailureAfterEachTestCases { + + @AfterEach + void afterEach() { + fail("after each"); + } + + @Test + @ExpectedToFail + void test() { + Assumptions.assumeTrue(false, "custom assumption message"); + } + + } + + static class ExpectedToFailFailureThenAbortAfterEachTestCases { + + @AfterEach + void afterEach() { + Assumptions.assumeTrue(false, "custom assumption message"); + } + + @Test + @ExpectedToFail + void test() { + fail("failed"); + } + + } + + /** + * {@link BeforeAll} should not be considered by {@link ExpectedToFail}. + */ + static class ExpectedToFailFailureBeforeAllTestCases { + + @BeforeAll + static void beforeAll() { + fail("before all"); + } + + @Test + @ExpectedToFail + void test() { + } + + } + + /** + * {@link AfterAll} should not be considered by {@link ExpectedToFail}. + */ + static class ExpectedToFailFailureAfterAllTestCases { + + @AfterAll + static void afterAll() { + fail("after all"); + } + + @Test + @ExpectedToFail + void test() { + } + + } + +} diff --git a/src/test/java/org/junitpioneer/testkit/assertion/PioneerAssert.java b/src/test/java/org/junitpioneer/testkit/assertion/PioneerAssert.java index 4eaba06f8..c1496e905 100644 --- a/src/test/java/org/junitpioneer/testkit/assertion/PioneerAssert.java +++ b/src/test/java/org/junitpioneer/testkit/assertion/PioneerAssert.java @@ -88,84 +88,100 @@ public void hasNoReportEntries() { @Override public TestCaseStartedAssert hasSingleStartedTest() { - return assertSingleTest(actual.testEvents().started().count()); + Events events = actual.testEvents(); + assertSingleTest(events.started()); + // Don't filter started() here; this would prevent further assertions on test outcome on + // returned assert because outcome is reported for the FINISHED event, not the STARTED one + return new TestCaseAssertBase(events); } @Override public TestCaseFailureAssert hasSingleFailedTest() { - return assertSingleTest(actual.testEvents().failed().count()); + return assertSingleTest(actual.testEvents().failed()); } @Override public void hasSingleAbortedTest() { - assertSingleTest(actual.testEvents().aborted().count()); + assertSingleTest(actual.testEvents().aborted()); } @Override public void hasSingleSucceededTest() { - assertSingleTest(actual.testEvents().succeeded().count()); + assertSingleTest(actual.testEvents().succeeded()); } @Override public void hasSingleSkippedTest() { - assertSingleTest(actual.testEvents().skipped().count()); + assertSingleTest(actual.testEvents().skipped()); } @Override public TestCaseStartedAssert hasSingleDynamicallyRegisteredTest() { - return assertSingleTest(actual.testEvents().dynamicallyRegistered().count()); + Events events = actual.testEvents(); + assertSingleTest(events.dynamicallyRegistered()); + // Don't filter dynamicallyRegistered() here; this would prevent further assertions on test outcome on + // returned assert because outcome is reported for the FINISHED event, not the DYNAMIC_TEST_REGISTERED one + return new TestCaseAssertBase(events); } - private TestCaseAssertBase assertSingleTest(long numberOfTestsWithOutcome) { + private TestCaseAssertBase assertSingleTest(Events events) { try { - Assertions.assertThat(numberOfTestsWithOutcome).isEqualTo(1); - return new TestCaseAssertBase(actual.testEvents()); + Assertions.assertThat(events.count()).isEqualTo(1); } catch (AssertionError error) { getAllExceptions(actual.allEvents()).forEach(error::addSuppressed); throw error; } + return new TestCaseAssertBase(events); } @Override public TestCaseStartedAssert hasSingleStartedContainer() { - return assertSingleContainer(actual.containerEvents().started().count()); + Events events = actual.containerEvents(); + assertSingleTest(events.started()); + // Don't filter started() here; this would prevent further assertions on container outcome on + // returned assert because outcome is reported for the FINISHED event, not the STARTED one + return new TestCaseAssertBase(events); } @Override public TestCaseFailureAssert hasSingleFailedContainer() { - return assertSingleContainer(actual.containerEvents().failed().count()); + return assertSingleContainer(actual.containerEvents().failed()); } @Override public void hasSingleAbortedContainer() { - assertSingleContainer(actual.containerEvents().aborted().count()); + assertSingleContainer(actual.containerEvents().aborted()); } @Override public void hasSingleSucceededContainer() { - assertSingleContainer(actual.containerEvents().succeeded().count()); + assertSingleContainer(actual.containerEvents().succeeded()); } @Override public void hasSingleSkippedContainer() { - assertSingleContainer(actual.containerEvents().skipped().count()); + assertSingleContainer(actual.containerEvents().skipped()); } @Override public TestCaseStartedAssert hasSingleDynamicallyRegisteredContainer() { - return assertSingleContainer(actual.containerEvents().dynamicallyRegistered().count()); + Events events = actual.containerEvents(); + assertSingleContainer(events.dynamicallyRegistered()); + // Don't filter dynamicallyRegistered() here; this would prevent further assertions on container outcome on + // returned assert because outcome is reported for the FINISHED event, not the DYNAMIC_TEST_REGISTERED one + return new TestCaseAssertBase(events); } - private TestCaseAssertBase assertSingleContainer(long numberOfContainersWithOutcome) { + private TestCaseAssertBase assertSingleContainer(Events events) { try { - Assertions.assertThat(numberOfContainersWithOutcome).isEqualTo(1); - return new TestCaseAssertBase(actual.containerEvents()); + Assertions.assertThat(events.count()).isEqualTo(1); } catch (AssertionError error) { getAllExceptions(actual.allEvents()).forEach(error::addSuppressed); throw error; } + return new TestCaseAssertBase(events); } @Override diff --git a/src/test/java/org/junitpioneer/testkit/assertion/TestCaseAssertBase.java b/src/test/java/org/junitpioneer/testkit/assertion/TestCaseAssertBase.java index d2d32692d..ba7b2c763 100644 --- a/src/test/java/org/junitpioneer/testkit/assertion/TestCaseAssertBase.java +++ b/src/test/java/org/junitpioneer/testkit/assertion/TestCaseAssertBase.java @@ -20,11 +20,12 @@ import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.testkit.engine.Events; +import org.junitpioneer.testkit.assertion.single.TestCaseAbortedAssert; import org.junitpioneer.testkit.assertion.single.TestCaseFailureAssert; import org.junitpioneer.testkit.assertion.single.TestCaseStartedAssert; class TestCaseAssertBase extends AbstractPioneerAssert - implements TestCaseStartedAssert, TestCaseFailureAssert { + implements TestCaseStartedAssert, TestCaseFailureAssert, TestCaseAbortedAssert { TestCaseAssertBase(Events events) { super(events, TestCaseAssertBase.class, 1); @@ -45,7 +46,7 @@ public AbstractThrowableAssert withExceptionInstance @Override public TestCaseFailureAssert whichFailed() { assertThat(actual.failed().count()).isEqualTo(1); - return this; + return new TestCaseAssertBase(actual.failed()); } @Override @@ -54,8 +55,9 @@ public void whichSucceeded() { } @Override - public void whichAborted() { + public TestCaseAbortedAssert whichAborted() { assertThat(actual.aborted().count()).isEqualTo(1); + return new TestCaseAssertBase(actual.aborted()); } @Override @@ -78,7 +80,6 @@ private Throwable getRequiredThrowable() { private Optional throwable() { return actual - .failed() .stream() .findFirst() .flatMap(fail -> fail.getPayload(TestExecutionResult.class)) diff --git a/src/test/java/org/junitpioneer/testkit/assertion/single/TestCaseAbortedAssert.java b/src/test/java/org/junitpioneer/testkit/assertion/single/TestCaseAbortedAssert.java new file mode 100644 index 000000000..dfe41cda2 --- /dev/null +++ b/src/test/java/org/junitpioneer/testkit/assertion/single/TestCaseAbortedAssert.java @@ -0,0 +1,28 @@ +/* + * Copyright 2016-2022 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * http://www.eclipse.org/legal/epl-v20.html + */ + +package org.junitpioneer.testkit.assertion.single; + +import org.assertj.core.api.AbstractThrowableAssert; + +/** + * Used to assert a single aborted container or test. + */ +public interface TestCaseAbortedAssert { + + /** + * Asserts that the test/container was aborted because of a specific type of exception. + * + * @param exceptionType the expected type of the thrown exception + * @return an {@link AbstractThrowableAssert} for further assertions + */ + AbstractThrowableAssert withExceptionInstanceOf(Class exceptionType); + +} diff --git a/src/test/java/org/junitpioneer/testkit/assertion/single/TestCaseStartedAssert.java b/src/test/java/org/junitpioneer/testkit/assertion/single/TestCaseStartedAssert.java index 5cd695aed..685fc92ca 100644 --- a/src/test/java/org/junitpioneer/testkit/assertion/single/TestCaseStartedAssert.java +++ b/src/test/java/org/junitpioneer/testkit/assertion/single/TestCaseStartedAssert.java @@ -23,8 +23,10 @@ public interface TestCaseStartedAssert { /** * Asserts that the test/container was aborted. + * + * @return a {@link TestCaseAbortedAssert} for further assertions. */ - void whichAborted(); + TestCaseAbortedAssert whichAborted(); /** * Asserts that the test/container has failed.