py36+ tests are definition ordered [v2]

changelog/5196.feature.rst

@@ -0,0 +1,3 @@
+Tests are now ordered by definition order in more cases.
+
+In a class hierarchy, tests from base classes are now consistently ordered before tests defined on thier subclasses (reverse MRO order).

src/_pytest/python.py

@@ -407,15 +407,18 @@ def collect(self) -> Iterable[Union[nodes.Item, nodes.Collector]]:
         if not getattr(self.obj, "__test__", True):
             return []
 
-        # NB. we avoid random getattrs and peek in the __dict__ instead
-        # (XXX originally introduced from a PyPy need, still true?)
+        # Avoid random getattrs and peek in the __dict__ instead.
         dicts = [getattr(self.obj, "__dict__", {})]
         for basecls in self.obj.__class__.__mro__:
             dicts.append(basecls.__dict__)
+
+        # In each class, nodes should be definition ordered. Since Python 3.6,
+        # __dict__ is definition ordered.
         seen: Set[str] = set()
-        values: List[Union[nodes.Item, nodes.Collector]] = []
+        dict_values: List[List[Union[nodes.Item, nodes.Collector]]] = []
         ihook = self.ihook
         for dic in dicts:
+            values: List[Union[nodes.Item, nodes.Collector]] = []
             # Note: seems like the dict can change during iteration -
             # be careful not to remove the list() without consideration.
             for name, obj in list(dic.items()):
@@ -433,13 +436,14 @@ def collect(self) -> Iterable[Union[nodes.Item, nodes.Collector]]:
                     values.extend(res)
                 else:
                     values.append(res)
-
-        def sort_key(item):
-            fspath, lineno, _ = item.reportinfo()
-            return (str(fspath), lineno)
-
-        values.sort(key=sort_key)
-        return values
+            dict_values.append(values)
+
+        # Between classes in the class hierarchy, reverse-MRO order -- nodes
+        # inherited from base classes should come before subclasses.
+        result = []
+        for values in reversed(dict_values):
+            result.extend(values)
+        return result
 
     def _genfunctions(self, name: str, funcobj) -> Iterator["Function"]:
         modulecol = self.getparent(Module)

testing/python/collect.py

@@ -770,6 +770,36 @@ def test_a(y):
         assert len(colitems) == 2
         assert [item.name for item in colitems] == ["test_b", "test_a"]
 
+    def test_ordered_by_definition_order(self, pytester: Pytester) -> None:
+        pytester.makepyfile(
+            """\
+            class Test1:
+                def test_foo(): pass
+                def test_bar(): pass
+            class Test2:
+                def test_foo(): pass
+                test_bar = Test1.test_bar
+            class Test3(Test2):
+                def test_baz(): pass
+            """
+        )
+        result = pytester.runpytest("--collect-only")
+        result.stdout.fnmatch_lines(
+            [
+                "*Class Test1*",
+                "*Function test_foo*",
+                "*Function test_bar*",
+                "*Class Test2*",
+                # previously the order was flipped due to Test1.test_bar reference
+                "*Function test_foo*",
+                "*Function test_bar*",
+                "*Class Test3*",
+                "*Function test_foo*",
+                "*Function test_bar*",
+                "*Function test_baz*",
+            ]
+        )
+
 
 class TestConftestCustomization:
     def test_pytest_pycollect_module(self, pytester: Pytester) -> None: