Enable Dataprovider failures to be considered.

Closes #217 Enable a data provider to propagate its failure And be considered as a test failure. This can be done at an individual data provider level by using the “@dataProvider” annotation’s new attribute “propagateFailureAsTestFailure” This can be done at the entire TestNG level by using the configuration parameter “-propagateDataProviderFailureAsTestFailure true” Also regrouped the data provider test into single test class.
testng-team · Apr 13, 2022 · b5b3e1d · b5b3e1d
1 parent f136315
commit b5b3e1d
Show file tree

Hide file tree

Showing 31 changed files with 224 additions and 163 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,5 +24,5 @@ test-output-tests
 testng.iml
 z_build
 .DS_Store
-
+outputDir/
 **/Version.java
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,5 +1,6 @@
 Current
 7.6.0
+Fixed: GITHUB-217: Configure TestNG to fail when there's a failure in data provider (Krishnan Mahadevan)
 Fixed: GITHUB-2743: SuiteRunner could not be initial by default Configuration (Nan Liang)
 Fixed: GITHUB-2729: beforeConfiguration() listener method should be invoked for skipped configurations as well(Nan Liang)
 Fixed: assertEqualsNoOrder for Collection and Iterators size check was missing (Adam Kaczmarek)

diff --git a/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java b/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java
@@ -25,4 +25,9 @@ public interface IDataProviderMethod {
 
   /** @return Which indices to run from this data provider, default: all. */
   List<Integer> getIndices();
+
+  /** @return Whether failures in data providers should be treated as test failures */
+  default boolean propagateFailureAsTestFailure() {
+    return false;
+  }
 }
diff --git a/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java b/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java
@@ -48,4 +48,11 @@
    * @return the value
    */
   int[] indices() default {};
+
+  /**
+   * Helps TestNG decide if it should treat data provider failures as test failures.
+   *
+   * @return the value
+   */
+  boolean propagateFailureAsTestFailure() default false;
 }
diff --git a/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java b/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java
@@ -21,4 +21,10 @@ public interface IDataProviderAnnotation extends IAnnotation {
   List<Integer> getIndices();
 
   void setIndices(List<Integer> indices);
+
+  /** Have TestNG consider failures in data provider methods as test failures. */
+  void propagateFailureAsTestFailure();
+
+  /** @return - <code>true</code>If data provider failures should be propagated as test failures */
+  boolean isPropagateFailureAsTestFailure();
 }
diff --git a/testng-core/src/main/java/org/testng/CommandLineArgs.java b/testng-core/src/main/java/org/testng/CommandLineArgs.java
@@ -251,4 +251,12 @@ public class CommandLineArgs {
       description =
           "Should TestNG report all iterations of a data driven test as individual skips, in-case of upstream failures.")
   public Boolean includeAllDataDrivenTestsWhenSkipping = false;
+
+  public static final String PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE =
+      "-propagateDataProviderFailureAsTestFailure";
+
+  @Parameter(
+      names = PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE,
+      description = "Should TestNG consider failures in Data Providers  as test failures.")
+  public Boolean propagateDataProviderFailureAsTestFailure = false;
 }
diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java
@@ -15,6 +15,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -596,6 +597,14 @@ public boolean getReportAllDataDrivenTestsAsSkipped() {
     return this.m_configuration.getReportAllDataDrivenTestsAsSkipped();
   }
 
+  public void propagateDataProviderFailureAsTestFailure() {
+    this.m_configuration.propagateDataProviderFailureAsTestFailure();
+  }
+
+  public boolean isPropagateDataProviderFailureAsTestFailure() {
+    return this.m_configuration.isPropagateDataProviderFailureAsTestFailure();
+  }
+
   /**
    * Set the suites file names to be run by this TestNG object. This method tries to load and parse
    * the specified TestNG suite xml files. If a file is missing, it is ignored.
@@ -1421,6 +1430,8 @@ public static TestNG privateMain(String[] argv, ITestListener listener) {
    * @param cla The command line parameters
    */
   protected void configure(CommandLineArgs cla) {
+    Optional.ofNullable(cla.propagateDataProviderFailureAsTestFailure)
+        .ifPresent(value -> propagateDataProviderFailureAsTestFailure());
     setReportAllDataDrivenTestsAsSkipped(cla.includeAllDataDrivenTestsWhenSkipping);
     if (cla.verbose != null) {
       setVerbose(cla.verbose);

diff --git a/testng-core/src/main/java/org/testng/internal/Configuration.java b/testng-core/src/main/java/org/testng/internal/Configuration.java
@@ -35,6 +35,8 @@ public class Configuration implements IConfiguration {
 
   private boolean includeAllDataDrivenTestsWhenSkipping;
 
+  private boolean propagateDataProviderFailureAsTestFailure;
+
   public Configuration() {
     init(new JDK15AnnotationFinder(new DefaultAnnotationTransformer()));
   }
@@ -156,4 +158,14 @@ public void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTest
   public boolean getReportAllDataDrivenTestsAsSkipped() {
     return this.includeAllDataDrivenTestsWhenSkipping;
   }
+
+  @Override
+  public void propagateDataProviderFailureAsTestFailure() {
+    this.propagateDataProviderFailureAsTestFailure = true;
+  }
+
+  @Override
+  public boolean isPropagateDataProviderFailureAsTestFailure() {
+    return propagateDataProviderFailureAsTestFailure;
+  }
 }
diff --git a/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java b/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java
@@ -42,4 +42,9 @@ public boolean isParallel() {
   public List<Integer> getIndices() {
     return annotation.getIndices();
   }
+
+  @Override
+  public boolean propagateFailureAsTestFailure() {
+    return annotation.isPropagateFailureAsTestFailure();
+  }
 }
diff --git a/testng-core/src/main/java/org/testng/internal/IConfiguration.java b/testng-core/src/main/java/org/testng/internal/IConfiguration.java
@@ -55,4 +55,8 @@ default void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTes
   default boolean getReportAllDataDrivenTestsAsSkipped() {
     return false;
   }
+
+  void propagateDataProviderFailureAsTestFailure();
+
+  boolean isPropagateDataProviderFailureAsTestFailure();
 }
diff --git a/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java b/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java
@@ -9,6 +9,7 @@ public class DataProviderAnnotation extends BaseAnnotation implements IDataProvi
   private String m_name;
   private boolean m_parallel;
   private List<Integer> m_indices;
+  private boolean m_bubbleUpFailures = false;
 
   @Override
   public boolean isParallel() {
@@ -39,4 +40,14 @@ public List<Integer> getIndices() {
   public void setIndices(List<Integer> indices) {
     m_indices = indices;
   }
+
+  @Override
+  public void propagateFailureAsTestFailure() {
+    m_bubbleUpFailures = true;
+  }
+
+  @Override
+  public boolean isPropagateFailureAsTestFailure() {
+    return m_bubbleUpFailures;
+  }
 }
diff --git a/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java b/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java
@@ -424,6 +424,9 @@ private IAnnotation createDataProviderTag(Method method, Annotation a) {
     }
     result.setParallel(c.parallel());
     result.setIndices(Ints.asList(c.indices()));
+    if (c.propagateFailureAsTestFailure()) {
+      result.propagateFailureAsTestFailure();
+    }
 
     return result;
   }

diff --git a/testng-core/src/main/java/org/testng/internal/invokers/ParameterHandler.java b/testng-core/src/main/java/org/testng/internal/invokers/ParameterHandler.java
@@ -3,7 +3,9 @@
 import static org.testng.internal.Parameters.MethodParameters;
 
 import java.util.Map;
+import java.util.Optional;
 import org.testng.DataProviderHolder;
+import org.testng.IDataProviderMethod;
 import org.testng.ITestContext;
 import org.testng.ITestNGMethod;
 import org.testng.ITestObjectFactory;
@@ -88,7 +90,12 @@ private ParameterBag handleParameters(
       }
 
       ITestResult result = TestResult.newTestResultWithCauseAs(testMethod, testContext, cause);
-      return new ParameterBag(result);
+
+      boolean bubbleUpFailure =
+          Optional.ofNullable(testMethod.getDataProviderMethod())
+              .map(IDataProviderMethod::propagateFailureAsTestFailure)
+              .orElse(false);
+      return new ParameterBag(result, bubbleUpFailure);
     }
   }
 
@@ -99,15 +106,17 @@ private ParameterBag handleParameters(
   static class ParameterBag {
     final ParameterHolder parameterHolder;
     final ITestResult errorResult;
+    boolean bubbleUpFailures = false;
 
     ParameterBag(ParameterHolder parameterHolder) {
       this.parameterHolder = parameterHolder;
       this.errorResult = null;
     }
 
-    ParameterBag(ITestResult errorResult) {
+    ParameterBag(ITestResult errorResult, boolean bubbleUpFailures) {
       this.parameterHolder = null;
       this.errorResult = errorResult;
+      this.bubbleUpFailures = bubbleUpFailures;
     }
 
     boolean hasErrors() {
@@ -119,5 +128,9 @@ boolean runInParallel() {
           && (parameterHolder.origin == ParameterHolder.ParameterOrigin.ORIGIN_DATA_PROVIDER
               && parameterHolder.dataProviderHolder.isParallel()));
     }
+
+    boolean isBubbleUpFailures() {
+      return bubbleUpFailures;
+    }
   }
 }
diff --git a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java
@@ -924,7 +924,11 @@ public int invoke(int invCount) {
       if (bag.hasErrors()) {
         ITestResult tr = bag.errorResult;
         Throwable throwable = Objects.requireNonNull(tr).getThrowable();
-        if (throwable instanceof TestNGException) {
+        boolean bubbleUpFailures =
+            m_configuration.isPropagateDataProviderFailureAsTestFailure()
+                || bag.isBubbleUpFailures();
+
+        if (throwable instanceof TestNGException || bubbleUpFailures) {
           tr.setStatus(ITestResult.FAILURE);
           m_notifier.addFailedTest(arguments.getTestMethod(), tr);
         } else {

diff --git a/testng-core/src/test/java/test/dataprovider/DataProviderTest.java b/testng-core/src/test/java/test/dataprovider/DataProviderTest.java
@@ -7,13 +7,19 @@
 import java.util.Collections;
 import java.util.List;
 import org.assertj.core.api.Condition;
+import org.assertj.core.api.SoftAssertions;
 import org.testng.Assert;
+import org.testng.IDataProviderMethod;
+import org.testng.ITestNGMethod;
 import org.testng.ITestResult;
 import org.testng.TestListenerAdapter;
 import org.testng.TestNG;
+import org.testng.TestNGException;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 import org.testng.internal.reflect.MethodMatcherException;
+import org.testng.xml.XmlClass;
+import org.testng.xml.XmlTest;
 import test.InvokedMethodNameListener;
 import test.SimpleBaseTest;
 import test.dataprovider.issue1691.DataProviderDefinitionAtClassLevelAndNoTestMethodUsage;
@@ -23,6 +29,12 @@
 import test.dataprovider.issue1691.withinheritance.ChildClassHasFullDefinitionOfDataProviderAtClassLevel;
 import test.dataprovider.issue1691.withinheritance.ChildClassHasPartialDefinitionOfDataProviderAtClassLevel;
 import test.dataprovider.issue1691.withinheritance.ChildClassWithNoDataProviderInformationInTestMethod;
+import test.dataprovider.issue1987.BaseClassSample;
+import test.dataprovider.issue1987.DataProviderInBaseClassSample;
+import test.dataprovider.issue1987.DataProviderInDifferentClassSample;
+import test.dataprovider.issue1987.DataProviderInSameClassSample;
+import test.dataprovider.issue1987.DataProviderTrackingListener;
+import test.dataprovider.issue2504.SampleTestCaseListener;
 import test.dataprovider.issue2565.Data;
 import test.dataprovider.issue2565.SampleTestUsingConsumer;
 import test.dataprovider.issue2565.SampleTestUsingFunction;
@@ -406,4 +418,97 @@ public void retryWithDataProvider() {
     assertThat(tla.getFailedTests()).size().isEqualTo(1);
     assertThat(tla.getSkippedTests()).size().isEqualTo(2);
   }
+
+  @Test(description = "GITHUB-217", expectedExceptions = TestNGException.class)
+  public void ensureTestNGThrowsExceptionWhenAllTestsAreSkipped() {
+    TestNG testng = create(test.dataprovider.issue217.TestClassSample.class);
+    testng.toggleFailureIfAllTestsWereSkipped(true);
+    testng.run();
+  }
+
+  @Test(description = "GITHUB-217")
+  public void ensureTestNGFailsDueToDataProviderFailure() {
+    TestNG testng = create(test.dataprovider.issue217.TestClassSample.class);
+    testng.propagateDataProviderFailureAsTestFailure();
+    testng.run();
+    assertThat(testng.getStatus()).isEqualTo(1);
+  }
+
+  @Test(description = "GITHUB-217")
+  public void ensureTestNGFailsDueToDataProviderFailure2() {
+    TestNG testng = create(test.dataprovider.issue217.AnotherTestClassSample.class);
+    testng.run();
+    assertThat(testng.getStatus()).isEqualTo(1);
+  }
+
+  @Test(description = "GITHUB-2255")
+  public void ensureDataProviderValuesAreVisibleToConfigMethods() {
+    TestNG testNG = create(test.dataprovider.issue2255.TestClassSample.class);
+    testNG.run();
+    assertThat(test.dataprovider.issue2255.TestClassSample.data).containsExactly(100, 200);
+  }
+
+  @Test(dataProvider = "testData", description = "GITHUB-1987")
+  public void extractDataProviderInfoWhenDpResidesInSameClass(
+      Class<?> clazz, boolean performInstanceCheck, Class<?> dataProviderClass) {
+    TestNG testng = create(clazz);
+    DataProviderTrackingListener listener = new DataProviderTrackingListener();
+    testng.addListener(listener);
+    testng.run();
+    ITestNGMethod method = listener.getResult().getMethod();
+    IDataProviderMethod dpm = method.getDataProviderMethod();
+    assertThat(dpm).isNotNull();
+    if (performInstanceCheck) {
+      assertThat(dpm.getInstance()).isEqualTo(method.getInstance());
+    }
+    assertThat(dpm.getMethod().getName()).isEqualTo("getData");
+    assertThat(dpm.getInstance().getClass()).isEqualTo(dataProviderClass);
+  }
+
+  @DataProvider(name = "testData")
+  public Object[][] getTestData() {
+    return new Object[][] {
+      {DataProviderInSameClassSample.class, true, DataProviderInSameClassSample.class},
+      {DataProviderInBaseClassSample.class, true, DataProviderInBaseClassSample.class},
+      {DataProviderInDifferentClassSample.class, false, BaseClassSample.class}
+    };
+  }
+
+  @Test(description = "GITHUB-2267")
+  public void ensureDynamicRetryAnalyzersAreHonouredForDataDrivenTest() {
+    TestNG testng = create(test.dataprovider.issue2267.TestClassSample.class);
+    TestListenerAdapter tla = new TestListenerAdapter();
+    testng.addListener(tla);
+    testng.run();
+    assertThat(tla.getFailedTests()).size().isEqualTo(1);
+    assertThat(tla.getSkippedTests()).size().isEqualTo(1);
+  }
+
+  @Test(description = "GITHUB-2327")
+  public void ensureDataProviderParametersAreAlwaysAvailableForListeners() {
+    TestNG testng = create(test.dataprovider.issue2327.TestClassSample.class);
+    TestListenerAdapter tla = new TestListenerAdapter();
+    testng.addListener(tla);
+    testng.run();
+
+    assertThat(tla.getSkippedTests().size()).isEqualTo(2);
+    SoftAssertions assertions = new SoftAssertions();
+
+    for (ITestResult skippedTest : tla.getSkippedTests()) {
+      assertions.assertThat(skippedTest.getParameters()).isNotEmpty();
+    }
+    assertions.assertAll();
+  }
+
+  @Test(description = "GITHUB-2504")
+  public void ensureParametersCopiedOnConfigFailures() {
+    XmlTest xmltest = createXmlTest("2504_suite", "2504_test");
+    xmltest.setXmlClasses(
+        Collections.singletonList(new XmlClass(test.dataprovider.issue2504.TestClassSample.class)));
+    TestNG testNG = create(Collections.singletonList(xmltest.getSuite()));
+    SampleTestCaseListener listener = new SampleTestCaseListener();
+    testNG.addListener(listener);
+    testNG.run();
+    assertThat(listener.getParameters()).containsExactlyElementsOf(Arrays.asList(1, 2, 3, 4, 5));
+  }
 }
diff --git a/...er/issue1987/DataProviderInBaseClass.java → ...ue1987/DataProviderInBaseClassSample.java b/...er/issue1987/DataProviderInBaseClass.java → ...ue1987/DataProviderInBaseClassSample.java
@@ -2,7 +2,7 @@
 
 import org.testng.annotations.Test;
 
-public class DataProviderInBaseClass extends BaseClassSample {
+public class DataProviderInBaseClassSample extends BaseClassSample {
 
   @Test(dataProvider = "dp")
   public void testMethod(int i) {}

diff --git a/...sue1987/DataProviderInDifferentClass.java → ...7/DataProviderInDifferentClassSample.java b/...sue1987/DataProviderInDifferentClass.java → ...7/DataProviderInDifferentClassSample.java
@@ -2,7 +2,7 @@
 
 import org.testng.annotations.Test;
 
-public class DataProviderInDifferentClass {
+public class DataProviderInDifferentClassSample {
 
   @Test(dataProvider = "dp", dataProviderClass = BaseClassSample.class)
   public void testMethod(int i) {}

diff --git a/...er/issue1987/DataProviderInSameClass.java → ...ue1987/DataProviderInSameClassSample.java b/...er/issue1987/DataProviderInSameClass.java → ...ue1987/DataProviderInSameClassSample.java
@@ -3,7 +3,7 @@
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
-public class DataProviderInSameClass {
+public class DataProviderInSameClassSample {
 
   @Test(dataProvider = "dp")
   public void testMethod(int i) {}