Ability to retry a data provider during failures

Closes #2819

CHANGES.txt

@@ -1,4 +1,5 @@
 Current
+Fixed: GITHUB-2819: Ability to retry a data provider in case of failures (Krishnan Mahadevan)
 Fixed: GITHUB-2308: StringIndexOutOfBoundsException in findClassesInPackage - Surefire/Maven - JDK 11 fails (Krishnan Mahadevan)
 Fixed: GITHUB:2788: TestResult.isSuccess() is TRUE when test fails due to expectedExceptions (Krishnan Mahadevan)
 Fixed: GITHUB-2800: Running Test Classes with Inherited @Factory and @DataProvider Annotated Non-Static Methods Fail (Krishnan Mahadevan)

testng-core-api/src/main/java/org/testng/IDataProviderMethod.java

@@ -30,4 +30,12 @@ public interface IDataProviderMethod {
   default boolean propagateFailureAsTestFailure() {
     return false;
   }
+
+  /**
+   * @return - An Class which implements {@link IRetryDataProvider} and which can be used to retry a
+   *     data provider.
+   */
+  default Class<? extends IRetryDataProvider> retryUsing() {
+    return IRetryDataProvider.DisableDataProviderRetries.class;
+  }
 }

testng-core-api/src/main/java/org/testng/IRetryDataProvider.java

@@ -0,0 +1,21 @@
+package org.testng;
+
+/** Represents the ability to retry a data provider. */
+public interface IRetryDataProvider {
+
+  /**
+   * @param dataProvider - The {@link IDataProviderMethod} object which represents the data provider
+   *     to be invoked.
+   * @return - <code>true</code> if the data provider should be invoked again.
+   */
+  boolean retry(IDataProviderMethod dataProvider);
+
+  /** A dummy implementation which disables retrying of a failed data provider. */
+  class DisableDataProviderRetries implements IRetryDataProvider {
+
+    @Override
+    public boolean retry(IDataProviderMethod dataProvider) {
+      return false;
+    }
+  }
+}

testng-core-api/src/main/java/org/testng/annotations/DataProvider.java

@@ -5,6 +5,7 @@
 import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
+import org.testng.IRetryDataProvider;
 
 /**
  * Mark a method as supplying data for a test method.
@@ -55,4 +56,11 @@
    * @return the value
    */
   boolean propagateFailureAsTestFailure() default false;
+
+  /**
+   * @return - An Class which implements {@link IRetryDataProvider} and which can be used to retry a
+   *     data provider.
+   */
+  Class<? extends IRetryDataProvider> retryUsing() default
+      IRetryDataProvider.DisableDataProviderRetries.class;
 }

testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java

@@ -1,6 +1,7 @@
 package org.testng.annotations;
 
 import java.util.List;
+import org.testng.IRetryDataProvider;
 
 /** Encapsulate the @DataProvider / @testng.data-provider annotation */
 public interface IDataProviderAnnotation extends IAnnotation {
@@ -27,4 +28,16 @@ public interface IDataProviderAnnotation extends IAnnotation {
 
   /** @return - <code>true</code>If data provider failures should be propagated as test failures */
   boolean isPropagateFailureAsTestFailure();
+
+  /**
+   * @param retry - A Class that implements {@link IRetryDataProvider} and which can be used to
+   *     retry a data provider.
+   */
+  void setRetryUsing(Class<? extends IRetryDataProvider> retry);
+
+  /**
+   * @return - An Class which implements {@link IRetryDataProvider} and which can be used to retry a
+   *     data provider.
+   */
+  Class<? extends IRetryDataProvider> retryUsing();
 }

testng-core/src/main/java/org/testng/internal/DataProviderMethod.java

@@ -3,6 +3,7 @@
 import java.lang.reflect.Method;
 import java.util.List;
 import org.testng.IDataProviderMethod;
+import org.testng.IRetryDataProvider;
 import org.testng.annotations.IDataProviderAnnotation;
 
 /** Represents an @{@link org.testng.annotations.DataProvider} annotated method. */
@@ -47,4 +48,9 @@ public List<Integer> getIndices() {
   public boolean propagateFailureAsTestFailure() {
     return annotation.isPropagateFailureAsTestFailure();
   }
+
+  @Override
+  public Class<? extends IRetryDataProvider> retryUsing() {
+    return annotation.retryUsing();
+  }
 }

testng-core/src/main/java/org/testng/internal/FilteredParameters.java

@@ -0,0 +1,62 @@
+package org.testng.internal;
+
+import java.util.Iterator;
+import java.util.List;
+import org.testng.ITestNGMethod;
+import org.testng.TestNGException;
+
+class FilteredParameters implements Iterator<Object[]> {
+
+  private int index = 0;
+  private boolean hasWarn = false;
+  private final Iterator<Object[]> parameters;
+  private final ITestNGMethod testMethod;
+  private final String dataProviderName;
+  private final List<Integer> indices;
+
+  public FilteredParameters(
+      Iterator<Object[]> parameters,
+      ITestNGMethod testMethod,
+      String dataProviderName,
+      List<Integer> indices) {
+    this.parameters = parameters;
+    this.testMethod = testMethod;
+    this.dataProviderName = dataProviderName;
+    this.indices = indices;
+  }
+
+  @Override
+  public boolean hasNext() {
+    if (index == 0 && !parameters.hasNext() && !hasWarn) {
+      hasWarn = true;
+      String msg =
+          String.format(
+              "The test method '%s' will be skipped since its "
+                  + "data provider '%s' "
+                  + "returned an empty array or iterator. ",
+              testMethod.getQualifiedName(), dataProviderName);
+      Utils.warn(msg);
+    }
+    return parameters.hasNext();
+  }
+
+  @Override
+  public Object[] next() {
+    testMethod.setParameterInvocationCount(index);
+    Object[] next = parameters.next();
+    if (next == null) {
+      throw new TestNGException("Parameters must not be null");
+    }
+    if (!indices.isEmpty() && !indices.contains(index)) {
+      // Skip parameters
+      next = null;
+    }
+    index++;
+    return next;
+  }
+
+  @Override
+  public void remove() {
+    throw new UnsupportedOperationException("remove");
+  }
+}

testng-core/src/main/java/org/testng/internal/Parameters.java

@@ -6,11 +6,11 @@
 import java.lang.reflect.Modifier;
 import java.lang.reflect.Parameter;
 import java.util.*;
-import javax.annotation.Nullable;
 import org.testng.DataProviderHolder;
 import org.testng.IDataProviderInterceptor;
 import org.testng.IDataProviderListener;
 import org.testng.IDataProviderMethod;
+import org.testng.IRetryDataProvider;
 import org.testng.ITestClass;
 import org.testng.ITestContext;
 import org.testng.ITestNGMethod;
@@ -147,7 +147,7 @@ public static Object[] createConfigurationParameters(
       Method m,
       Map<String, String> params,
       Object[] parameterValues,
-      @Nullable ITestNGMethod currentTestMethod,
+      ITestNGMethod currentTestMethod,
       IAnnotationFinder finder,
       XmlSuite xmlSuite,
       ITestContext ctx,
@@ -618,34 +618,33 @@ private static IDataProviderMethod findDataProvider(
 
     for (Method m : ClassHelper.getAvailableMethods(cls)) {
       IDataProviderAnnotation dp = finder.findAnnotation(m, IDataProviderAnnotation.class);
-      if (null != dp && name.equals(getDataProviderName(dp, m))) {
-        Object instanceToUse;
-        if (shouldBeStatic && (m.getModifiers() & Modifier.STATIC) == 0) {
-          IObjectDispenser dispenser = Dispenser.newInstance(objectFactory);
-          BasicAttributes basic = new BasicAttributes(clazz, dataProviderClass);
-          CreationAttributes attributes = new CreationAttributes(context, basic, null);
-          instanceToUse = dispenser.dispense(attributes);
-        } else {
-          instanceToUse = instance;
-        }
-        // Not a static method but no instance exists, then create new one if possible
-        if ((m.getModifiers() & Modifier.STATIC) == 0 && instanceToUse == null) {
-          try {
-            instanceToUse = objectFactory.newInstance(cls);
-          } catch (TestNGException e) {
-            instanceToUse = null;
-          }
+      boolean proceed = null != dp && name.equals(getDataProviderName(dp, m));
+      if (!proceed) {
+        continue;
+      }
+      Object instanceToUse = instance;
+      if (shouldBeStatic && (m.getModifiers() & Modifier.STATIC) == 0) {
+        IObjectDispenser dispenser = Dispenser.newInstance(objectFactory);
+        BasicAttributes basic = new BasicAttributes(clazz, dataProviderClass);
+        CreationAttributes attributes = new CreationAttributes(context, basic, null);
+        instanceToUse = dispenser.dispense(attributes);
+      }
+      // Not a static method but no instance exists, then create new one if possible
+      if ((m.getModifiers() & Modifier.STATIC) == 0 && instanceToUse == null) {
+        try {
+          instanceToUse = objectFactory.newInstance(cls);
+        } catch (TestNGException ignored) {
         }
+      }
 
-        if (result != null) {
-          throw new TestNGException("Found two providers called '" + name + "' on " + cls);
-        }
+      if (result != null) {
+        throw new TestNGException("Found two providers called '" + name + "' on " + cls);
+      }
 
-        if (isDynamicDataProvider) {
-          result = new DataProviderMethodRemovable(instanceToUse, m, dp);
-        } else {
-          result = new DataProviderMethod(instanceToUse, m, dp);
-        }
+      if (isDynamicDataProvider) {
+        result = new DataProviderMethodRemovable(instanceToUse, m, dp);
+      } else {
+        result = new DataProviderMethod(instanceToUse, m, dp);
       }
     }
 
@@ -780,31 +779,46 @@ public static ParameterHolder handleParameters(
         String n = "param" + i;
         allParameterNames.put(n, n);
       }
-
-      for (IDataProviderListener dataProviderListener : holder.getListeners()) {
-        dataProviderListener.beforeDataProviderExecution(
-            dataProviderMethod, testMethod, methodParams.context);
+      Class<?> retryClass = dataProviderMethod.retryUsing();
+      boolean shouldRetry = !retryClass.equals(IRetryDataProvider.DisableDataProviderRetries.class);
+      IRetryDataProvider retry = null;
+      if (shouldRetry) {
+        IObjectDispenser dispenser = Dispenser.newInstance(objectFactory);
+        BasicAttributes basic = new BasicAttributes(testMethod.getTestClass(), retryClass);
+        CreationAttributes attributes = new CreationAttributes(methodParams.context, basic, null);
+        retry = (IRetryDataProvider) dispenser.dispense(attributes);
       }
 
-      Iterator<Object[]> initParams;
-      try {
-        initParams =
-            MethodInvocationHelper.invokeDataProvider(
-                dataProviderMethod
-                    .getInstance(), /* a test instance or null if the data provider is static*/
-                dataProviderMethod.getMethod(),
-                testMethod,
-                methodParams.context,
-                fedInstance,
-                annotationFinder);
-      } catch (RuntimeException e) {
-        for (IDataProviderListener each : holder.getListeners()) {
-          each.onDataProviderFailure(testMethod, methodParams.context, e);
+      Iterator<Object[]> initParams = null;
+      do {
+
+        for (IDataProviderListener dataProviderListener : holder.getListeners()) {
+          dataProviderListener.beforeDataProviderExecution(
+              dataProviderMethod, testMethod, methodParams.context);
         }
-        throw e;
-      }
 
-      final Iterator<Object[]> parameters = initParams;
+        try {
+          initParams =
+              MethodInvocationHelper.invokeDataProvider(
+                  dataProviderMethod
+                      .getInstance(), /* a test instance or null if the data provider is static*/
+                  dataProviderMethod.getMethod(),
+                  testMethod,
+                  methodParams.context,
+                  fedInstance,
+                  annotationFinder);
+          shouldRetry = false;
+        } catch (RuntimeException e) {
+          for (IDataProviderListener each : holder.getListeners()) {
+            each.onDataProviderFailure(testMethod, methodParams.context, e);
+          }
+          if (shouldRetry) {
+            shouldRetry = retry.retry(dataProviderMethod);
+          } else {
+            throw e;
+          }
+        }
+      } while (shouldRetry);
 
       for (IDataProviderListener dataProviderListener : holder.getListeners()) {
         dataProviderListener.afterDataProviderExecution(
@@ -817,45 +831,7 @@ public static ParameterHolder handleParameters(
       allIndices.addAll(dataProviderMethod.getIndices());
 
       Iterator<Object[]> filteredParameters =
-          new Iterator<Object[]>() {
-            int index = 0;
-            boolean hasWarn = false;
-
-            @Override
-            public boolean hasNext() {
-              if (index == 0 && !parameters.hasNext() && !hasWarn) {
-                hasWarn = true;
-                String msg =
-                    String.format(
-                        "The test method '%s' will be skipped since its "
-                            + "data provider '%s' "
-                            + "returned an empty array or iterator. ",
-                        testMethod.getQualifiedName(), dataProviderMethod.getName());
-                Utils.warn(msg);
-              }
-              return parameters.hasNext();
-            }
-
-            @Override
-            public Object[] next() {
-              testMethod.setParameterInvocationCount(index);
-              Object[] next = parameters.next();
-              if (next == null) {
-                throw new TestNGException("Parameters must not be null");
-              }
-              if (!allIndices.isEmpty() && !allIndices.contains(index)) {
-                // Skip parameters
-                next = null;
-              }
-              index++;
-              return next;
-            }
-
-            @Override
-            public void remove() {
-              throw new UnsupportedOperationException("remove");
-            }
-          };
+          new FilteredParameters(initParams, testMethod, dataProviderMethod.getName(), allIndices);
 
       testMethod.setMoreInvocationChecker(filteredParameters::hasNext);
       for (IDataProviderInterceptor interceptor : holder.getInterceptors()) {
@@ -919,18 +895,6 @@ public static Object[] injectParameters(
     return matcher.getConformingArguments();
   }
 
-  public static Object[] getParametersFromIndex(Iterator<Object[]> parametersValues, int index) {
-    while (parametersValues.hasNext()) {
-      Object[] parameters = parametersValues.next();
-
-      if (index == 0) {
-        return parameters;
-      }
-      index--;
-    }
-    return null;
-  }
-
   /** A parameter passing helper class. */
   public static class MethodParameters {
     private final Map<String, String> xmlParameters;