Add @Flaky annotation to be used with @Test instead of @RetryingTest

src/main/java/org/junitpioneer/jupiter/Flaky.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2016-2022 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter;
+
+import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.parallel.Execution;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.ANNOTATION_TYPE, ElementType.METHOD })
+// the extension is inherently thread-unsafe (has to wait for one execution before starting the next),
+// so it forces execution of all retries onto the same thread
+@Execution(SAME_THREAD)
+@ExtendWith(FlakyExtension.class)
+@TestTemplate
+public @interface Flaky {
+
+	int value() default 0;
+
+	String name() default "[{index}]";
+
+}

src/main/java/org/junitpioneer/jupiter/FlakyExtension.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright 2016-2022 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter;
+
+import static java.lang.String.format;
+import static org.junit.platform.commons.support.AnnotationSupport.findAnnotation;
+
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtensionConfigurationException;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ExtensionContext.Namespace;
+import org.junit.jupiter.api.extension.TestExecutionExceptionHandler;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+import org.junit.jupiter.api.extension.TestWatcher;
+import org.junitpioneer.internal.PioneerAnnotationUtils;
+import org.junitpioneer.internal.TestNameFormatter;
+import org.opentest4j.TestAbortedException;
+
+class FlakyExtension implements TestExecutionExceptionHandler, TestTemplateInvocationContextProvider,
+		ExecutionCondition, TestWatcher {
+
+	private static final Namespace NAMESPACE = Namespace.create(FlakyExtension.class);
+
+	@Override
+	public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
+		var root = context.getRoot();
+		if (root.getStore(NAMESPACE).get(repeatId(context)) == null) {
+			// repeat id is only present in the store for the test template
+			root.getStore(NAMESPACE).put(uniqueId(context), Status.TEST_FAILED);
+			throw new TestAbortedException("The test has failed and will be retried.", throwable);
+		} else {
+			// The test method being present means this is the test template
+			int i = root.getStore(NAMESPACE).get(repeatId(context), int.class);
+			if (i > 0) {
+				root.getStore(NAMESPACE).put(repeatId(context), i - 1);
+				throw new TestAbortedException("The test has failed and will be retried", throwable);
+			}
+			root.getStore(NAMESPACE).remove(repeatId(context));
+			root.getStore(NAMESPACE).put(uniqueId(context), Status.TEST_TEMPLATE_FAILED);
+			throw throwable;
+		}
+	}
+
+	@Override
+	public void testSuccessful(ExtensionContext context) {
+		context.getRoot().getStore(NAMESPACE).put(uniqueId(context), Status.TEST_SUCCESSFUL);
+	}
+
+	@Override
+	public boolean supportsTestTemplate(ExtensionContext context) {
+		return PioneerAnnotationUtils.isAnnotationPresent(context, Flaky.class);
+	}
+
+	@Override
+	public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(ExtensionContext context) {
+		Flaky flaky = findAnnotation(context.getTestMethod(), Flaky.class)
+				.orElseThrow(() -> new IllegalStateException(
+					"Flaky extension was invoked but @Flaky annotation is not present."));
+		final var formatter = new TestNameFormatter(flaky.name(), context.getDisplayName(), Flaky.class);
+		int value = flaky.value();
+		if (value <= 0)
+			throw new ExtensionConfigurationException("value must be greater than 0.");
+		if (context.getRoot().getStore(NAMESPACE).get(uniqueId(context)) != null) {
+			// normal test ran, one less test template
+			value -= 1;
+		}
+		context.getRoot().getStore(NAMESPACE).put(repeatId(context), value - 1);
+		return Stream.generate(() -> new FlakyTestInvocationContext(formatter)).limit(value).map(Function.identity());
+	}
+
+	// This is invoked BEFORE test templates get created!
+	@Override
+	public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
+		var root = context.getRoot();
+		var testRunResult = root.getStore(NAMESPACE).get(uniqueId(context));
+		if (testRunResult == Status.TEST_SUCCESSFUL)
+			return ConditionEvaluationResult.disabled("There was a successful test, subsequent runs can be disabled.");
+		if (testRunResult == Status.TEST_TEMPLATE_FAILED)
+			return ConditionEvaluationResult.disabled("Test template ran and failed, test is skipped.");
+		return ConditionEvaluationResult.enabled("Test can run.");
+	}
+
+	private static String uniqueId(ExtensionContext context) {
+		return context.getRoot().getUniqueId() + format("/[method:%s()]", context.getRequiredTestMethod().getName());
+	}
+
+	private static String repeatId(ExtensionContext context) {
+		return uniqueId(context) + "$repeats";
+	}
+
+	private enum Status {
+		TEST_FAILED, TEST_TEMPLATE_FAILED, TEST_SUCCESSFUL // or test template successful
+	}
+
+}

src/main/java/org/junitpioneer/jupiter/FlakyTestInvocationContext.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2016-2022 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter;
+
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junitpioneer.internal.TestNameFormatter;
+
+class FlakyTestInvocationContext implements TestTemplateInvocationContext {
+
+	private final TestNameFormatter formatter;
+
+	FlakyTestInvocationContext(TestNameFormatter formatter) {
+		this.formatter = formatter;
+	}
+
+	@Override
+	public String getDisplayName(int invocationIndex) {
+		return formatter.format(invocationIndex);
+	}
+
+}

src/test/java/org/junitpioneer/jupiter/FlakyExtensionTests.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2016-2022 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter;
+
+import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS;
+import static org.junitpioneer.testkit.PioneerTestKit.executeTestMethod;
+import static org.junitpioneer.testkit.assertion.PioneerAssert.assertThat;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.ExtensionConfigurationException;
+import org.junitpioneer.testkit.ExecutionResults;
+
+public class FlakyExtensionTests {
+
+	@Test
+	@DisplayName("completely disregards Flaky annotation if initial Test is successful")
+	void disregarded() {
+		ExecutionResults results = executeTestMethod(FlakyTestCases.class, "negativeValuePassing");
+
+		assertThat(results).hasNumberOfSucceededContainers(2);
+		assertThat(results).hasSingleSucceededTest();
+	}
+
+	@Test
+	@DisplayName("throws an ExtensionConfigurationException for negative value if initial Test is failing")
+	void invalidValue() {
+		ExecutionResults results = executeTestMethod(FlakyTestCases.class, "negativeValueFailing");
+
+		assertThat(results).hasNumberOfAbortedTests(1);
+		assertThat(results).hasSingleFailedContainer().withExceptionInstanceOf(ExtensionConfigurationException.class);
+	}
+
+	@Test
+	@DisplayName("failing test annotated with @Flaky and value 3 should run three times, abort twice and fail once")
+	void alwaysFailing() {
+		ExecutionResults results = executeTestMethod(FlakyTestCases.class, "alwaysFail");
+
+		assertThat(results).hasNumberOfAbortedTests(2);
+		assertThat(results)
+				.hasSingleFailedTest()
+				.withExceptionInstanceOf(IllegalStateException.class)
+				.hasMessage("Always failing");
+	}
+
+	@Test
+	@DisplayName("once failing test annotated with @Flaky and value 3 should run three times, abort once then succeed")
+	void failsOnlyFirstTime() {
+		ExecutionResults results = executeTestMethod(FlakyTestCases.class, "failsOnlyOnFirstInvocation");
+
+		assertThat(results).hasSingleSucceededTest();
+		assertThat(results).hasSingleSkippedTest();
+		assertThat(results).hasSingleAbortedTest();
+	}
+
+	@TestInstance(PER_CLASS)
+	static class FlakyTestCases {
+
+		private int executionCount;
+
+		@Test
+		@Flaky(value = -1)
+		void negativeValuePassing() {
+		}
+
+		@Test
+		@Flaky(-1)
+		void negativeValueFailing() {
+			Assertions.fail();
+		}
+
+		@Test
+		@Flaky(3)
+		void alwaysFail() {
+			throw new IllegalStateException("Always failing");
+		}
+
+		@Test
+		@Flaky(3)
+		void failsOnlyOnFirstInvocation() {
+			executionCount++;
+			if (executionCount == 1) {
+				throw new IllegalArgumentException();
+			}
+		}
+
+	}
+
+}