From 0659796a199e58fd6520a4a806bfff04afd8488e Mon Sep 17 00:00:00 2001 From: Warren Date: Sun, 17 Mar 2024 16:24:20 +1100 Subject: [PATCH 01/11] test(fixtures-per-test): exclude pseudo fixture output Adds a test suite to validate that --fixtures-per-test behaves as intuitively as possible for new users by excluding counter-intuitive output: the pseudo fixtures that result from directly parametrizing a test with ``@pytest.mark.parametrize``. The test suite further validates that fixtures which have been parametrized, either by direct or indirect means, are retained in the --fixtures-per-test output. --- testing/python/show_fixtures_per_test.py | 74 +++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/testing/python/show_fixtures_per_test.py b/testing/python/show_fixtures_per_test.py index f756dca41c7..f38bc4e2feb 100644 --- a/testing/python/show_fixtures_per_test.py +++ b/testing/python/show_fixtures_per_test.py @@ -1,7 +1,7 @@ from _pytest.pytester import Pytester -def test_no_items_should_not_show_output(pytester: Pytester) -> None: +def test_should_show_no_ouput_when_zero_items(pytester: Pytester) -> None: result = pytester.runpytest("--fixtures-per-test") result.stdout.no_fnmatch_line("*fixtures used by*") assert result.ret == 0 @@ -252,3 +252,75 @@ def test_arg1(arg1): " Docstring content that extends into a third paragraph.", ] ) + + +def test_should_not_show_pseudo_fixtures(pytester: Pytester) -> None: + """A fixture is considered pseudo if it was directly created using the + @pytest.mark.parametrize decorator as part of internal pytest mechanisms + to manage batch execution. These fixtures should not be included in the + output because they don't satisfy user expectations for how fixtures are + created and used.""" + + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("x", [1]) + def test_pseudo_fixture(x): + pass + """ + ) + result = pytester.runpytest("--fixtures-per-test", p) + result.stdout.no_fnmatch_line("*fixtures used by*") + assert result.ret == 0 + + +def test_should_show_parametrized_fixtures_used_by_test(pytester: Pytester) -> None: + """A fixture with parameters should be included if it was created using + the @pytest.fixture decorator, including those that are indirectly + parametrized.""" + p = pytester.makepyfile( + ''' + import pytest + + @pytest.fixture(params=['a', 'b']) + def directly(request): + """parametrized fixture""" + return request.param + + @pytest.fixture + def indirectly(request): + """indirectly parametrized fixture""" + return request.param + + def test_parametrized_fixture(directly): + pass + + @pytest.mark.parametrize("indirectly", ["a", "b"], indirect=True) + def test_indirectly_parametrized(indirectly): + pass + ''' + ) + result = pytester.runpytest("--fixtures-per-test", p) + assert result.ret == 0 + + expected_matches_for_parametrized_test = [ + "*fixtures used by test_parametrized_fixture*", + "*(test_should_show_parametrized_fixtures_used_by_test.py:14)*", + "directly -- test_should_show_parametrized_fixtures_used_by_test.py:4", + " parametrized fixture", + ] + + expected_matches_for_indirectly_parametrized_test = [ + "*fixtures used by test_indirectly_parametrized*", + "*(test_should_show_parametrized_fixtures_used_by_test.py:17)*", + "indirectly -- test_should_show_parametrized_fixtures_used_by_test.py:9", + " indirectly parametrized fixture", + ] + + expected_matches = ( + expected_matches_for_parametrized_test * 2 + + expected_matches_for_indirectly_parametrized_test * 2 + ) + + result.stdout.fnmatch_lines(expected_matches) From 2fdee7b062134e938d9e335f5534a3640b493ef9 Mon Sep 17 00:00:00 2001 From: Warren Date: Sun, 17 Mar 2024 16:27:15 +1100 Subject: [PATCH 02/11] refactor: utilise existing FixtureDef import --- src/_pytest/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 3242d517ea0..8e5bc3703b4 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -1482,7 +1482,7 @@ def _validate_if_using_arg_names( def _find_parametrized_scope( argnames: Sequence[str], - arg2fixturedefs: Mapping[str, Sequence[fixtures.FixtureDef[object]]], + arg2fixturedefs: Mapping[str, Sequence[FixtureDef[object]]], indirect: Union[bool, Sequence[str]], ) -> Scope: """Find the most appropriate scope for a parametrized call based on its arguments. From b9e7493f8a3cdbc3dd5351e6f36f22586cf5d7c6 Mon Sep 17 00:00:00 2001 From: Warren Date: Sun, 17 Mar 2024 16:30:26 +1100 Subject: [PATCH 03/11] feat(fixtures-per-test): exclude pseudo fixtures from output Addresses issue #11295 by excluding from the --fixtures-per-test output any 'pseudo fixture' that results from directly parametrizating a test with ``@pytest.mark.parametrize``. The justification for removing these fixtures from the report is that a) They are unintuitive. Their appearance in the fixtures-per-test report confuses new users because the fixtures created via ``@pytest.mark.parametrize`` do not confrom to the expectations established in the documentation; namely, that fixtures are - richly reusable - provide setup/teardown features - created via the ``@pytest.fixture` decorator b) They are an internal implementation detail. It is not the explicit goal of the direct parametrization mark to create a fixture; instead, pytest's internals leverages the fixture system to achieve the explicit goal: a succinct batch execution syntax. Consequently, exposing the fixtures that implement the batch execution behaviour reveal more about pytest's internals than they do about the user's own design choices and test dependencies. --- src/_pytest/python.py | 76 ++++++++++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 8e5bc3703b4..4da96cbf1ff 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -1534,6 +1534,48 @@ def _pretty_fixture_path(invocation_dir: Path, func) -> str: return bestrelpath(invocation_dir, loc) +def _get_fixtures_per_test(test: nodes.Item) -> Optional[List[FixtureDef[object]]]: + """Returns all fixtures used by the test item except for a) those + created by direct parametrization with ``@pytest.mark.parametrize`` and + b) those accessed dynamically with ``request.getfixturevalue``. + + The justification for excluding fixtures created by direct + parametrization is that their appearance in a report would surprise + users currently learning about fixtures, as they do not conform to the + documented characteristics of fixtures (reusable, providing + setup/teardown features, and created via the ``@pytest.fixture`` + decorator). + + In other words, an internal detail that leverages the fixture system + to batch execute tests should not be exposed in a report that identifies + which fixtures a user is using in their tests. + """ + # Contains information on the fixtures the test item requests + # statically, if any. + fixture_info: Optional[FuncFixtureInfo] = getattr(test, "_fixtureinfo", None) + if fixture_info is None: + # The given test item does not statically request any fixtures. + return [] + + # In the transitive closure of fixture names required by the item + # through autouse, function parameter or @userfixture mechanisms, + # multiple overrides may have occured; for this reason, the fixture + # names are matched to a sequence of FixtureDefs. + name2fixturedefs = fixture_info.name2fixturedefs + fixturedefs = [ + # The final override, which is the one the test item will utilise, + # is stored in the final position of the sequence; therefore, we + # take the FixtureDef of the final override and add it to the list. + # + # If there wasn't an ovveride, the final item will simply be the + # first item, as required. + fixturedefs[-1] + for argname, fixturedefs in sorted(name2fixturedefs.items()) + if argname not in _get_direct_parametrize_args(test) + ] + return fixturedefs + + def show_fixtures_per_test(config): from _pytest.main import wrap_session @@ -1552,7 +1594,21 @@ def get_best_relpath(func) -> str: loc = getlocation(func, invocation_dir) return bestrelpath(invocation_dir, Path(loc)) - def write_fixture(fixture_def: fixtures.FixtureDef[object]) -> None: + def write_item(item: nodes.Item) -> None: + fixturedefs = _get_fixtures_per_test(item) + if not fixturedefs: + # This test item does not use any fixtures. + # Do not write anything. + return + + tw.line() + tw.sep("-", f"fixtures used by {item.name}") + tw.sep("-", f"({get_best_relpath(item.function)})") # type: ignore[attr-defined] + + for fixturedef in fixturedefs: + write_fixture(fixturedef) + + def write_fixture(fixture_def: FixtureDef[object]) -> None: argname = fixture_def.argname if verbose <= 0 and argname.startswith("_"): return @@ -1568,24 +1624,6 @@ def write_fixture(fixture_def: fixtures.FixtureDef[object]) -> None: else: tw.line(" no docstring available", red=True) - def write_item(item: nodes.Item) -> None: - # Not all items have _fixtureinfo attribute. - info: Optional[FuncFixtureInfo] = getattr(item, "_fixtureinfo", None) - if info is None or not info.name2fixturedefs: - # This test item does not use any fixtures. - return - tw.line() - tw.sep("-", f"fixtures used by {item.name}") - # TODO: Fix this type ignore. - tw.sep("-", f"({get_best_relpath(item.function)})") # type: ignore[attr-defined] - # dict key not used in loop but needed for sorting. - for _, fixturedefs in sorted(info.name2fixturedefs.items()): - assert fixturedefs is not None - if not fixturedefs: - continue - # Last item is expected to be the one used by the test item. - write_fixture(fixturedefs[-1]) - for session_item in session.items: write_item(session_item) From a2450647c1ac297c8d3eb3f4fd6b10a6037bffe1 Mon Sep 17 00:00:00 2001 From: Warren Date: Sun, 17 Mar 2024 23:12:17 +1100 Subject: [PATCH 04/11] refactor(test): rename test functions --- testing/python/show_fixtures_per_test.py | 35 ++++++++++++------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/testing/python/show_fixtures_per_test.py b/testing/python/show_fixtures_per_test.py index f38bc4e2feb..92b074ce072 100644 --- a/testing/python/show_fixtures_per_test.py +++ b/testing/python/show_fixtures_per_test.py @@ -256,11 +256,10 @@ def test_arg1(arg1): def test_should_not_show_pseudo_fixtures(pytester: Pytester) -> None: """A fixture is considered pseudo if it was directly created using the - @pytest.mark.parametrize decorator as part of internal pytest mechanisms - to manage batch execution. These fixtures should not be included in the - output because they don't satisfy user expectations for how fixtures are - created and used.""" - + ``@pytest.mark.parametrize`` decorator as part of internal pytest + mechanisms (such as to manage batch execution). These fixtures should not + be included in the output because they don't satisfy user expectations for + how fixtures are created and used.""" p = pytester.makepyfile( """ import pytest @@ -282,45 +281,45 @@ def test_should_show_parametrized_fixtures_used_by_test(pytester: Pytester) -> N p = pytester.makepyfile( ''' import pytest - + @pytest.fixture(params=['a', 'b']) def directly(request): """parametrized fixture""" return request.param - + @pytest.fixture def indirectly(request): """indirectly parametrized fixture""" return request.param - - def test_parametrized_fixture(directly): + + def test_directly_parametrized_fixture(directly): pass - + @pytest.mark.parametrize("indirectly", ["a", "b"], indirect=True) - def test_indirectly_parametrized(indirectly): - pass + def test_indirectly_parametrized_fixture(indirectly): + pass ''' ) result = pytester.runpytest("--fixtures-per-test", p) assert result.ret == 0 - expected_matches_for_parametrized_test = [ - "*fixtures used by test_parametrized_fixture*", + expected_matches_for_directly_parametrized_fixture_test = [ + "*fixtures used by test_directly_parametrized_fixture*", "*(test_should_show_parametrized_fixtures_used_by_test.py:14)*", "directly -- test_should_show_parametrized_fixtures_used_by_test.py:4", " parametrized fixture", ] - expected_matches_for_indirectly_parametrized_test = [ - "*fixtures used by test_indirectly_parametrized*", + expected_matches_for_indirectly_parametrized_fixture_test = [ + "*fixtures used by test_indirectly_parametrized_fixture*", "*(test_should_show_parametrized_fixtures_used_by_test.py:17)*", "indirectly -- test_should_show_parametrized_fixtures_used_by_test.py:9", " indirectly parametrized fixture", ] expected_matches = ( - expected_matches_for_parametrized_test * 2 - + expected_matches_for_indirectly_parametrized_test * 2 + expected_matches_for_directly_parametrized_fixture_test * 2 + + expected_matches_for_indirectly_parametrized_fixture_test * 2 ) result.stdout.fnmatch_lines(expected_matches) From eced627f75e7101cb78e9d83350035c12d3cd56d Mon Sep 17 00:00:00 2001 From: Warren Date: Sun, 17 Mar 2024 23:13:55 +1100 Subject: [PATCH 05/11] fix(fixture-per-test): add missing import --- src/_pytest/python.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 4da96cbf1ff..24ae81c00fd 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -58,7 +58,7 @@ from _pytest.config import hookimpl from _pytest.config.argparsing import Parser from _pytest.deprecated import check_ispytest -from _pytest.fixtures import FixtureDef +from _pytest.fixtures import FixtureDef, _get_direct_parametrize_args from _pytest.fixtures import FixtureRequest from _pytest.fixtures import FuncFixtureInfo from _pytest.fixtures import get_scope_node @@ -1603,7 +1603,7 @@ def write_item(item: nodes.Item) -> None: tw.line() tw.sep("-", f"fixtures used by {item.name}") - tw.sep("-", f"({get_best_relpath(item.function)})") # type: ignore[attr-defined] + tw.sep("-", f"({get_best_relpath(item.function)})") # type: ignore[attr-defined] for fixturedef in fixturedefs: write_fixture(fixturedef) From 05b4b78b3ea964858f643edcbd28c7b317c50b46 Mon Sep 17 00:00:00 2001 From: Warren Date: Sun, 17 Mar 2024 23:21:21 +1100 Subject: [PATCH 06/11] doc: improve wording --- src/_pytest/python.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 24ae81c00fd..3f8808949eb 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -58,7 +58,8 @@ from _pytest.config import hookimpl from _pytest.config.argparsing import Parser from _pytest.deprecated import check_ispytest -from _pytest.fixtures import FixtureDef, _get_direct_parametrize_args +from _pytest.fixtures import _get_direct_parametrize_args +from _pytest.fixtures import FixtureDef from _pytest.fixtures import FixtureRequest from _pytest.fixtures import FuncFixtureInfo from _pytest.fixtures import get_scope_node @@ -1547,8 +1548,8 @@ def _get_fixtures_per_test(test: nodes.Item) -> Optional[List[FixtureDef[object] decorator). In other words, an internal detail that leverages the fixture system - to batch execute tests should not be exposed in a report that identifies - which fixtures a user is using in their tests. + to batch execute tests should not be exposed in a report intended to + summarise the user's fixture choices. """ # Contains information on the fixtures the test item requests # statically, if any. @@ -1567,7 +1568,7 @@ def _get_fixtures_per_test(test: nodes.Item) -> Optional[List[FixtureDef[object] # is stored in the final position of the sequence; therefore, we # take the FixtureDef of the final override and add it to the list. # - # If there wasn't an ovveride, the final item will simply be the + # If there wasn't an overide, the final item will simply be the # first item, as required. fixturedefs[-1] for argname, fixturedefs in sorted(name2fixturedefs.items()) From 2dd8e367ba4d0093af3caba8dcbc716db270ecdd Mon Sep 17 00:00:00 2001 From: Warren Date: Sun, 17 Mar 2024 23:31:39 +1100 Subject: [PATCH 07/11] chore: update changelog --- changelog/11295.improvement.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/11295.improvement.rst diff --git a/changelog/11295.improvement.rst b/changelog/11295.improvement.rst new file mode 100644 index 00000000000..299cff9fab8 --- /dev/null +++ b/changelog/11295.improvement.rst @@ -0,0 +1 @@ +Improved output of --fixtures-per-test by excluding internal, pseudo fixtures. From 56713ea2fe2d90fa3357d3233dcc7a770503a096 Mon Sep 17 00:00:00 2001 From: Warren Date: Sun, 17 Mar 2024 23:34:37 +1100 Subject: [PATCH 08/11] chore: add name to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 53f7a8c2a16..8e136384c58 100644 --- a/AUTHORS +++ b/AUTHORS @@ -423,6 +423,7 @@ Vlad Radziuk Vladyslav Rachek Volodymyr Kochetkov Volodymyr Piskun +Warren Markham Wei Lin Wil Cooley William Lee From e46fe6f55f850a4c25a562657487df96d4e05e8f Mon Sep 17 00:00:00 2001 From: Warren Date: Sun, 17 Mar 2024 23:48:53 +1100 Subject: [PATCH 09/11] doc: correct spelling --- src/_pytest/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 3f8808949eb..b089e650405 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -1568,7 +1568,7 @@ def _get_fixtures_per_test(test: nodes.Item) -> Optional[List[FixtureDef[object] # is stored in the final position of the sequence; therefore, we # take the FixtureDef of the final override and add it to the list. # - # If there wasn't an overide, the final item will simply be the + # If there wasn't an override, the final item will simply be the # first item, as required. fixturedefs[-1] for argname, fixturedefs in sorted(name2fixturedefs.items()) From accf57c8a6a3e442bd619ee3d27150b1fbf8772c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 17 Mar 2024 13:43:54 +0000 Subject: [PATCH 10/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- testing/python/show_fixtures_per_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/python/show_fixtures_per_test.py b/testing/python/show_fixtures_per_test.py index 92b074ce072..87e7a76f32d 100644 --- a/testing/python/show_fixtures_per_test.py +++ b/testing/python/show_fixtures_per_test.py @@ -256,9 +256,9 @@ def test_arg1(arg1): def test_should_not_show_pseudo_fixtures(pytester: Pytester) -> None: """A fixture is considered pseudo if it was directly created using the - ``@pytest.mark.parametrize`` decorator as part of internal pytest - mechanisms (such as to manage batch execution). These fixtures should not - be included in the output because they don't satisfy user expectations for + ``@pytest.mark.parametrize`` decorator as part of internal pytest + mechanisms (such as to manage batch execution). These fixtures should not + be included in the output because they don't satisfy user expectations for how fixtures are created and used.""" p = pytester.makepyfile( """ From c0a23aca06c97726d3246bbff0d39084c64fb21d Mon Sep 17 00:00:00 2001 From: Warren Date: Mon, 18 Mar 2024 07:51:00 +1100 Subject: [PATCH 11/11] refactor(test): make expected output explicit instead of calculated --- testing/python/show_fixtures_per_test.py | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/testing/python/show_fixtures_per_test.py b/testing/python/show_fixtures_per_test.py index 87e7a76f32d..38902f7ebb3 100644 --- a/testing/python/show_fixtures_per_test.py +++ b/testing/python/show_fixtures_per_test.py @@ -303,23 +303,23 @@ def test_indirectly_parametrized_fixture(indirectly): result = pytester.runpytest("--fixtures-per-test", p) assert result.ret == 0 - expected_matches_for_directly_parametrized_fixture_test = [ - "*fixtures used by test_directly_parametrized_fixture*", - "*(test_should_show_parametrized_fixtures_used_by_test.py:14)*", - "directly -- test_should_show_parametrized_fixtures_used_by_test.py:4", - " parametrized fixture", - ] - - expected_matches_for_indirectly_parametrized_fixture_test = [ - "*fixtures used by test_indirectly_parametrized_fixture*", - "*(test_should_show_parametrized_fixtures_used_by_test.py:17)*", - "indirectly -- test_should_show_parametrized_fixtures_used_by_test.py:9", - " indirectly parametrized fixture", - ] - - expected_matches = ( - expected_matches_for_directly_parametrized_fixture_test * 2 - + expected_matches_for_indirectly_parametrized_fixture_test * 2 + result.stdout.fnmatch_lines( + [ + "*fixtures used by test_directly_parametrized_fixture*", + "*(test_should_show_parametrized_fixtures_used_by_test.py:14)*", + "directly -- test_should_show_parametrized_fixtures_used_by_test.py:4", + " parametrized fixture", + "*fixtures used by test_directly_parametrized_fixture*", + "*(test_should_show_parametrized_fixtures_used_by_test.py:14)*", + "directly -- test_should_show_parametrized_fixtures_used_by_test.py:4", + " parametrized fixture", + "*fixtures used by test_indirectly_parametrized_fixture*", + "*(test_should_show_parametrized_fixtures_used_by_test.py:17)*", + "indirectly -- test_should_show_parametrized_fixtures_used_by_test.py:9", + " indirectly parametrized fixture", + "*fixtures used by test_indirectly_parametrized_fixture*", + "*(test_should_show_parametrized_fixtures_used_by_test.py:17)*", + "indirectly -- test_should_show_parametrized_fixtures_used_by_test.py:9", + " indirectly parametrized fixture", + ] ) - - result.stdout.fnmatch_lines(expected_matches)