diff --git a/pyproject.toml b/pyproject.toml index a57df0d2495..14a2511c4c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,7 @@ lint = [ ] test = [ "pytest>=6.0", + "pytest-xdist==3.5.0", "defusedxml>=0.7.1", # for secure XML/HTML parsing "cython>=3.0", "setuptools>=67.0", # for Cython compilation @@ -287,6 +288,7 @@ disallow_any_generics = false minversion = "6.0" addopts = [ "-ra", + "-p no:xdist", # disable xdist for now "--import-mode=prepend", # "--pythonwarnings=error", "--strict-config", diff --git a/sphinx/testing/_internal/__init__.py b/sphinx/testing/_internal/__init__.py new file mode 100644 index 00000000000..c87b0b8c613 --- /dev/null +++ b/sphinx/testing/_internal/__init__.py @@ -0,0 +1,6 @@ +"""This package contains implementation details for the Sphinx testing plugin. + +All modules in this package are considered an implementation detail +and any provided functionality can be altered, removed or introduce +breaking changes without prior notice. +""" diff --git a/sphinx/testing/_internal/cache.py b/sphinx/testing/_internal/cache.py new file mode 100644 index 00000000000..9bf05e02ad0 --- /dev/null +++ b/sphinx/testing/_internal/cache.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +__all__ = () + +import dataclasses +import itertools +import shutil +from io import StringIO +from typing import TYPE_CHECKING, TypedDict + +if TYPE_CHECKING: + from typing import Any + + from sphinx.testing.util import SphinxTestApp, SphinxTestAppWrapperForSkipBuilding + + +class _CacheEntry(TypedDict): + """Cached entry in a :class:`ModuleCache`.""" + + status: str + """The application's status output.""" + warning: str + """The application's warning output.""" + + +class _CacheFrame(TypedDict): + """The restored cached value.""" + + status: StringIO + """An I/O object initialized to the cached status output.""" + warning: StringIO + """An I/O object initialized to the cached warning output.""" + + +class ModuleCache: + __slots__ = ('_cache',) + + def __init__(self) -> None: + self._cache: dict[str, _CacheEntry] = {} + + def clear(self) -> None: + """Clear the cache.""" + self._cache.clear() + + def store(self, key: str, app: SphinxTestApp) -> None: + """Cache some attributes from *app* in the cache. + + :param key: The cache key (usually a ``shared_result``). + :param app: An application whose attributes are cached. + + The application's attributes being cached are: + + * The content of :attr:`~sphinx.testing.util.SphinxTestApp.status`. + * The content of :attr:`~sphinx.testing.util.SphinxTestApp.warning`. + """ + if key not in self._cache: + status, warning = app.status.getvalue(), app.warning.getvalue() + self._cache[key] = {'status': status, 'warning': warning} + + def restore(self, key: str) -> _CacheFrame | None: + """Reconstruct the cached attributes for *key*.""" + if key not in self._cache: + return None + + data = self._cache[key] + return {'status': StringIO(data['status']), 'warning': StringIO(data['warning'])} + + +@dataclasses.dataclass +class AppInfo: + """Information to report during the teardown phase of the ``app()`` fixture. + + The information is either rendered as a report section (for ``xdist`` + integration) or directly printed using a ``print`` statement. + """ + + builder: str + """The builder name.""" + + testroot_path: str | None + """The absolute path to the sources directory (if any).""" + shared_result: str | None + """The user-defined shared result (if any).""" + + srcdir: str + """The absolute path to the application's sources directory.""" + outdir: str + """The absolute path to the application's output directory.""" + + extras: dict[str, Any] = dataclasses.field(default_factory=dict, init=False) + """Attributes added by :func:`sphinx.testing.fixtures.app_test_info`.""" + + # fields below are updated when tearing down :func:`sphinx.testing.fixtures.app` + _messages: str = dataclasses.field(default='', init=False) + """The application's status messages (updated by the fixture).""" + _warnings: str = dataclasses.field(default='', init=False) + """The application's warnings messages (updated by the fixture).""" + + def update(self, app: SphinxTestApp | SphinxTestAppWrapperForSkipBuilding) -> None: + """Update the application's status and warning messages.""" + self._messages = app.status.getvalue() + self._warnings = app.warning.getvalue() + + def render(self, nodeid: str | None = None) -> str: + """Format the report as a string to print or render. + + :param nodeid: Optional node id to include in the report. + :return: The formatted information. + """ + config = [('builder', self.builder)] + if nodeid: + config.insert(0, ('test case', nodeid)) + + if self.testroot_path: + config.append(('testroot path', self.testroot_path)) + config.extend([('srcdir', self.srcdir), ('outdir', self.outdir)]) + config.extend((name, repr(value)) for name, value in self.extras.items()) + + tw, _ = shutil.get_terminal_size() + kw = 8 + max(len(name) for name, _ in config) + + lines = itertools.chain( + [f'{" environment ":-^{tw}}'], + (f'{name:{kw}s} {strvalue}' for name, strvalue in config), + [f'{" messages ":-^{tw}}', text] if (text := self._messages) else (), + [f'{" warnings ":-^{tw}}', text] if (text := self._warnings) else (), + ['=' * tw], + ) + return '\n'.join(lines) + + +# XXX: RemovedInSphinx90Warning +class LegacyModuleCache: # kept for legacy purposes + cache: dict[str, dict[str, str]] = {} + + def store( + self, key: str, app_: SphinxTestApp | SphinxTestAppWrapperForSkipBuilding + ) -> None: + if key in self.cache: + return + data = { + 'status': app_.status.getvalue(), + 'warning': app_.warning.getvalue(), + } + self.cache[key] = data + + def restore(self, key: str) -> dict[str, StringIO]: + if key not in self.cache: + return {} + + data = self.cache[key] + return { + 'status': StringIO(data['status']), + 'warning': StringIO(data['warning']), + } diff --git a/sphinx/testing/_internal/isolation.py b/sphinx/testing/_internal/isolation.py new file mode 100644 index 00000000000..e856b9569b1 --- /dev/null +++ b/sphinx/testing/_internal/isolation.py @@ -0,0 +1,46 @@ +"""Private module containing isolation-related objects and functionalities. + +Use literal strings or booleans to indicate isolation policies instead of +directly using :class:`Isolation` objects, unless it is used internally. +""" + +from __future__ import annotations + +__all__ = () + +from enum import IntEnum +from enum import auto as _auto +from typing import Literal, Union + + +class Isolation(IntEnum): + """Isolation policy for the testing application.""" + + minimal = _auto() + """Minimal isolation mode.""" + grouped = _auto() + """Similar to :attr:`always` but for parametrized tests.""" + always = _auto() + """Copy the original testroot to a unique sources and build directory.""" + + +IsolationPolicy = Union[bool, Literal['minimal', 'grouped', 'always']] +"""Allowed values for the isolation policy.""" + +NormalizableIsolation = Union[IsolationPolicy, Isolation] +"""Normalizable isolation value.""" + + +def normalize_isolation_policy(policy: NormalizableIsolation) -> Isolation: + """Normalize isolation policy into a :class:`Isolation` object.""" + if isinstance(policy, Isolation): + return policy + + if isinstance(policy, bool): + return Isolation.always if policy else Isolation.minimal + + if isinstance(policy, str) and hasattr(Isolation, policy): + return getattr(Isolation, policy) + + msg = f'unknown isolation policy: {policy!r}' + raise TypeError(msg) diff --git a/sphinx/testing/_internal/markers.py b/sphinx/testing/_internal/markers.py new file mode 100644 index 00000000000..a0fedea3bea --- /dev/null +++ b/sphinx/testing/_internal/markers.py @@ -0,0 +1,368 @@ +"""Private utililty functions for markers in :mod:`sphinx.testing.plugin`. + +This module is an implementation detail and any provided function +or class can be altered, removed or moved without prior notice. +""" + +from __future__ import annotations + +__all__ = () + +from pathlib import Path +from typing import TYPE_CHECKING, NamedTuple, TypedDict, cast + +import pytest + +from sphinx.testing._internal.isolation import Isolation, normalize_isolation_policy +from sphinx.testing._internal.pytest_util import ( + check_mark_keywords, + check_mark_str_args, + format_mark_failure, + get_mark_parameters, + get_node_location, +) +from sphinx.testing._internal.util import ( + get_container_id, + get_location_id, + get_objects_checksum, + make_unique_id, +) + +if TYPE_CHECKING: + import os + from io import StringIO + from typing import Any + + from _pytest.nodes import Node as PytestNode + from typing_extensions import Required + + from sphinx.testing._internal.isolation import NormalizableIsolation + from sphinx.testing._internal.pytest_util import TestRootFinder + + +class SphinxMarkEnviron(TypedDict, total=False): + """Typed dictionary for the arguments of :func:`pytest.mark.sphinx`. + + For the :func:`!pytest.mark.sphinx` marker, we only allow keyword + arguments and not positional arguments except the builder name. + + Note that this differs from the :class:`~sphinx.testing.util.SphinxTestApp` + constructor which accepts both positional and keyword arguments; however + this is done as such so that it makes easier to check the marker itself. + """ + + buildername: str + srcdir: str + + confoverrides: dict[str, Any] + # using freshenv=True will be treated as equivalent to use isolate=True + # but in the future, we might want to deprecate this marker keyword in + # favor of "isolate" (that way, we don't need to maintain it) + freshenv: bool + warningiserror: bool + tags: list[str] + verbosity: int + parallel: int + keep_going: bool + + builddir: str + docutils_conf: str + + # added or updated fields + testroot: str | None + isolate: NormalizableIsolation + + +class SphinxInitKwargs(TypedDict, total=False): + """The type of the keyword arguments after processing. + + Such objects are constructed from :class:`SphinxMarkEnviron` objects. + """ + + # :class:`sphinx.application.Sphinx` positional arguments as keywords + buildername: Required[str] + """The deduced builder name.""" + # :class:`sphinx.application.Sphinx` required arguments + srcdir: Required[Path] + """Absolute path to the test sources directory. + + The uniqueness of this path depends on the isolation policy, + the location of the test and the application's configuration. + """ + # :class:`sphinx.application.Sphinx` optional arguments + confoverrides: dict[str, Any] | None + status: StringIO | None + warning: StringIO | None + freshenv: bool + warningiserror: bool + tags: list[str] | None + verbosity: int + parallel: int + keep_going: bool + # :class:`sphinx.testing.util.SphinxTestApp` optional arguments + builddir: Path | None + docutils_conf: str | None + # :class:`sphinx.testing.util.SphinxTestApp` extras arguments + isolate: Required[Isolation] + """The deduced isolation policy.""" + testroot: Required[str | None] + """The deduced testroot ID (possibly None if the default ID is not set).""" + testroot_path: Required[str | None] + """The absolute path to the testroot directory, if any.""" + shared_result: Required[str | None] + """The optional shared result ID.""" + + +class AppParams(NamedTuple): + """The processed arguments of :func:`pytest.mark.sphinx`. + + The *args* and *kwargs* values can be directly used as inputs + to the :class:`~sphinx.testing.util.SphinxTestApp` constructor. + """ + + args: list[Any] + """The constructor positional arguments, except ``buildername``.""" + kwargs: SphinxInitKwargs + """The constructor keyword arguments, including ``buildername``.""" + + +class AppLegacyParams(NamedTuple): + args: list[Any] + kwargs: dict[str, Any] + + +class TestParams(TypedDict): + """A view on the arguments of :func:`pytest.mark.test_params`.""" + + shared_result: str | None + + +def _get_sphinx_environ(node: PytestNode, default_builder: str) -> SphinxMarkEnviron: + args, kwargs = get_mark_parameters(node, 'sphinx') + + if len(args) > 1: + err = 'expecting at most one positional argument' + pytest.fail(format_mark_failure('sphinx', err)) + + env = cast(SphinxMarkEnviron, kwargs) + + if args: + buildername = args.pop() + if buildername != env.pop('buildername', buildername): + err = '%r has duplicated values' % 'buildername' + pytest.fail(format_mark_failure('sphinx', err)) + env['buildername'] = buildername + else: + buildername = env.setdefault('buildername', default_builder) + + if not buildername: + pytest.fail(format_mark_failure('sphinx', 'missing builder name')) + + check_mark_keywords('sphinx', SphinxMarkEnviron.__annotations__, env, node=node) + return env + + +def _get_test_srcdir( + srcdir: str | None, + testroot: str | None, + shared_result: str | None, +) -> str: + """Deduce the sources directory from the given arguments. + + :param srcdir: An optional explicit source directory name. + :param testroot: An optional testroot ID to use. + :param shared_result: An optional shared result ID. + :return: The sources directory name *srcdir* (non-empty string). + """ + check_mark_str_args('sphinx', srcdir=srcdir, testroot=testroot) + check_mark_str_args('test_params', shared_result=shared_result) + + if srcdir is not None: + # the srcdir is explicitly given, so we use this name + # and we do not bother to make it unique (the user is + # responsible for that !) + return srcdir + + if shared_result is not None: + # include the testroot id for visual purposes (unless it is + # not specified, which only occurs when there is no rootdir) + return f'{testroot}-{shared_result}' if testroot else shared_result + + if testroot is None: + # neither an explicit nor the default testroot ID is given + pytest.fail('missing %r or %r parameter' % ('testroot', 'srcdir')) + + return testroot + + +def _get_common_config_checksum( + # positional-only to avoid _get_config_checksum(name, **kwargs) + # raising a "duplicated values for 'buildername'" ValueError + buildername: str, + /, + *, + # The default values must be kept in sync with the constructor + # default values of :class:`sphinx.testing.util.SphinxTestApp` + # + # Note that 'srcdir' and 'builddir' are not used to construct + # the checksum since otherwise the checksum is unique (and we + # only want a uniqueness that only depend on common user-defined + # values). Similarly, 'status' and 'warning' are not used to + # construct the checksum they are stream objects in general. + confoverrides: dict[str, Any] | None = None, + freshenv: bool = False, + warningiserror: bool = False, + tags: list[str] | None = None, + verbosity: int = 0, + parallel: int = 0, + keep_going: bool = False, + # extra constructor argument + docutils_conf: str | None = None, + # ignored keyword arguments when computing the checksum + **_ignored: Any, +) -> int: + # fmt: off + return get_objects_checksum( + buildername, confoverrides=confoverrides, freshenv=freshenv, + warningiserror=warningiserror, tags=tags, verbosity=verbosity, + parallel=parallel, keep_going=keep_going, + # extra constructor arguments + docutils_conf=docutils_conf, + ) + # fmt: on + + +def process_sphinx( + node: PytestNode, + session_temp_dir: str | os.PathLike[str], + testroot_finder: TestRootFinder, + default_builder: str, + default_isolation: NormalizableIsolation, + shared_result: str | None, +) -> tuple[list[Any], SphinxInitKwargs]: + """Process the :func:`pytest.mark.sphinx` marker. + + :param node: The pytest node to parse. + :param session_temp_dir: The session temporary directory. + :param testroot_finder: The testroot finder object. + :param default_builder: The application default builder name. + :param default_isolation: The isolation default policy. + :param shared_result: An optional shared result ID. + :return: The application positional and keyword arguments. + """ + # process pytest.mark.sphinx + env = _get_sphinx_environ(node, default_builder) + + # deduce the testroot ID + testroot_id = env['testroot'] = env.get('testroot', testroot_finder.default) + # deduce the srcdir name (possibly explicitly given) + srcdir_name = env.get('srcdir', None) + srcdir = _get_test_srcdir(srcdir_name, testroot_id, shared_result) + is_unique_srcdir_id = srcdir_name is not None + + # deduce the isolation policy from freshenv if possible + freshenv: bool | None = env.pop('freshenv', None) + if freshenv is not None: + if 'isolate' in env: + err = '%r and %r are mutually exclusive' % ('freshenv', 'isolate') + pytest.fail(format_mark_failure('sphinx', err)) + + isolation = env['isolate'] = Isolation.always if freshenv else default_isolation + else: + freshenv = env['freshenv'] = False + + # deduce the final isolation policy + isolation = is_unique_srcdir_id or env.setdefault('isolate', default_isolation) + isolation = env['isolate'] = normalize_isolation_policy(isolation) + + # process the srcdir ID according to the isolation policy + if isolation is Isolation.always and srcdir_name is None: + # srcdir = XYZ-(RANDOM-UID) + srcdir = make_unique_id(srcdir) + is_unique_srcdir_id = True + elif isolation is Isolation.grouped: + if (location := get_node_location(node)) is None: + srcdir = make_unique_id(srcdir) + is_unique_srcdir_id = True + else: + # For a 'grouped' isolation, we want the same prefix (the deduced + # sources dierctory), but with a unique suffix based on the node + # location. In particular, parmetrized tests will have the same + # final ``srcdir`` value as they have the same location. + suffix = get_location_id(location) + # srcdir = XYZ-(RANDOM-UID) + srcdir = f'{srcdir}-{suffix}' + + if is_unique_srcdir_id: + # when the sources directory is known to be unique across + # all other tests, we do not include a namespace or checksum + namespace, checksum = '-', 0 + else: + namespace = get_container_id(node) + # Do a somewhat hash on configuration values to give a minimal protection + # against side-effects (two tests with the same configuration should have + # the same output; if they mess up with their sources directory, then they + # should be isolated accordingly). If there is a bug in the test suite, we + # can reduce the number of tests that can have dependencies by adding some + # isolation safeguards. + checksum = _get_common_config_checksum(env['buildername'], **env) + + kwargs = cast(SphinxInitKwargs, env) + kwargs['srcdir'] = Path(session_temp_dir, namespace, str(checksum), srcdir) + # ensure that the type of a possible 'builddir' argument is indeed a Path + kwargs['builddir'] = Path(builddir) if (builddir := env.get('builddir')) else None + kwargs['testroot_path'] = testroot_finder.find(testroot_id) + kwargs['shared_result'] = shared_result + return [], kwargs + + +def process_test_params(node: PytestNode) -> TestParams: + """Process the :func:`pytest.mark.test_params` marker. + + :param node: The pytest node to parse. + :return: The desired keyword arguments. + """ + ret = TestParams(shared_result=None) + if (m := node.get_closest_marker('test_params')) is None: + return ret + + if m.args: + pytest.fail(format_mark_failure('test_params', 'unexpected positional argument')) + + kwargs, allowed_keywords = m.kwargs, TestParams.__annotations__ + check_mark_keywords('test_params', allowed_keywords, kwargs, node=node, strict=True) + + if (shared_result_id := kwargs.get('shared_result', None)) is None: + # generate a random shared_result for @pytest.mark.test_params() + # based on either the location of node (so that it is the same + # when using @pytest.mark.parametrize()) + if (location := get_node_location(node)) is None: + shared_result_id = make_unique_id() + else: + shared_result_id = get_location_id(location) + + ret['shared_result'] = shared_result_id + return ret + + +def process_isolate(node: PytestNode, default: NormalizableIsolation) -> NormalizableIsolation: + """Process the :func:`pytest.mark.isolate` marker. + + :param node: The pytest node to parse. + :param default: The default isolation policy given by an external fixture. + :return: The isolation policy given by the marker. + """ + # try to find an isolation policy from the 'isolate' marker + if m := node.get_closest_marker('isolate'): + # do not allow keyword arguments + check_mark_keywords('isolate', [], m.kwargs, node=node, strict=True) + if not m.args: + # isolate() is equivalent to a full isolation + return Isolation.always + + if len(m.args) == 1: + return normalize_isolation_policy(m.args[0]) + + err = 'expecting at most one positional argument' + pytest.fail(format_mark_failure('isolate', err)) + return default diff --git a/sphinx/testing/_internal/pytest_util.py b/sphinx/testing/_internal/pytest_util.py new file mode 100644 index 00000000000..ef4d38883a9 --- /dev/null +++ b/sphinx/testing/_internal/pytest_util.py @@ -0,0 +1,413 @@ +"""Internal utility functions for interacting with pytest.""" + +from __future__ import annotations + +__all__ = () + +import os +import warnings +from contextlib import contextmanager +from typing import TYPE_CHECKING, Literal, TypeVar, overload + +import pytest +from _pytest.nodes import Node as PytestNode +from _pytest.nodes import get_fslocation_from_item + +from sphinx.testing._internal.warnings import MarkWarning, NodeWarning, SphinxTestingWarning + +if TYPE_CHECKING: + from collections.abc import Callable, Collection, Iterable, Iterator + from typing import Any, ClassVar, Final + + T = TypeVar('T') + DT = TypeVar('DT') + NodeType = TypeVar('NodeType', bound='PytestNode') + + +class TestRootFinder: + """Object responsible for finding the testroot files in *rootdir*. + + For instance:: + + finder = TestRootFinder('/foo/bar', 'test-', 'default') + + describes a testroot root directory at ``/foo/bar/``. The name of the + directories in ``/foo/bar/`` consist of a *prefix* and an *ID* (in this + case, the prefix is ``test-`` and the default *ID* is ``default``). + + >>> finder = TestRootFinder('/foo/bar', 'test-', 'default') + >>> finder.find() + '/foo/bar/test-default' + >>> finder.find('abc') + '/foo/bar/test-abc' + """ + + # This is needed to avoid this class being considered as a test by pytest. + __test__: ClassVar[Literal[False]] = False + + def __init__( + self, + path: str | os.PathLike[str] | None = None, + prefix: str | None = None, + default: str | None = None, + ) -> None: + """Construct a :class:`TestRootFinder` object. + + :param path: Optional non-empty root path containing the testroots. + :param prefix: Optional prefix to prepend to a testroot ID. + :param default: Optional non-empty string for a default testroot ID. + :raise ValueError: Empty strings are given instead of ``None``. + """ + for arg, val in (('path', path), ('default', default)): + if not val and val is not None: + msg = 'expecting a non-empty string or None for %r' + raise ValueError(msg % arg) + + self.path: str | None = os.fsdecode(path) if path else None + + assert prefix is None or isinstance(prefix, str) + self.prefix: str = prefix or '' + + assert default is None or isinstance(default, str) + self.default: str | None = default + + def find(self, testroot_id: str | None = None) -> str | None: + """Find the sources directory for a named or the default testroot. + + :param testroot_id: A testroot ID (non-prefixed string). + :return: The path to the testroot directory, if any. + """ + if not (path := self.path): + return None + + if not (testroot_id := testroot_id or self.default): + return None + + # upon construction, we ensured that 'prefix' is empty if None + return os.path.join(path, f'{self.prefix}{testroot_id}') + + +ScopeName = Literal['session', 'package', 'module', 'class', 'function'] +"""Pytest scopes.""" + +_NODE_TYPE_BY_SCOPE: Final[dict[ScopeName, type[PytestNode]]] = { + 'session': pytest.Session, + 'package': pytest.Package, + 'module': pytest.Module, + 'class': pytest.Class, + 'function': pytest.Function, +} + + +# fmt: off +@overload +def get_node_type_by_scope(scope: Literal['session']) -> type[pytest.Session]: ... # NoQA: E501, E704 +@overload +def get_node_type_by_scope(scope: Literal['package']) -> type[pytest.Package]: ... # NoQA: E501, E704 +@overload +def get_node_type_by_scope(scope: Literal['module']) -> type[pytest.Module]: ... # NoQA: E704 +@overload +def get_node_type_by_scope(scope: Literal['class']) -> type[pytest.Class]: ... # NoQA: E704 +@overload +def get_node_type_by_scope(scope: Literal['function']) -> type[pytest.Function]: ... # NoQA: E501, E704 +# fmt: on +def get_node_type_by_scope(scope: ScopeName) -> type[PytestNode]: # NoQA: E302 + """Get a pytest node type by its scope. + + :param scope: The scope name. + :return: The corresponding pytest node type. + """ + return _NODE_TYPE_BY_SCOPE[scope] + + +# fmt: off +@overload +def find_context(node: PytestNode, cond: Literal['session'], /, *, include_self: bool = ...) -> pytest.Session: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: Literal['package'], /, *, include_self: bool = ...) -> pytest.Package: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: Literal['module'], /, *, include_self: bool = ...) -> pytest.Module: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: Literal['class'], /, *, include_self: bool = ...) -> pytest.Class: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: Literal['function'], /, *, include_self: bool = ...) -> pytest.Function: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: ScopeName, /, *, include_self: bool = ...) -> PytestNode: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: Literal['session'], default: DT, /, *, include_self: bool = ...) -> pytest.Session | DT: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: Literal['package'], default: DT, /, *, include_self: bool = ...) -> pytest.Package | DT: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: Literal['module'], default: DT, /, *, include_self: bool = ...) -> pytest.Module | DT: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: Literal['class'], default: DT, /, *, include_self: bool = ...) -> pytest.Class | DT: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: Literal['function'], default: DT, /, *, include_self: bool = ...) -> pytest.Function | DT: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: ScopeName, default: DT, /, *, include_self: bool = ...) -> PytestNode | DT: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: type[NodeType], /, *, include_self: bool = ...) -> NodeType: ... # NoQA: E501, E704 +@overload +def find_context(node: PytestNode, cond: type[NodeType], default: DT, /, *, include_self: bool = ...) -> NodeType | DT: ... # NoQA: E501, E704 +# fmt: on +def find_context( # NoQA: E302 + node: Any, + cond: ScopeName | type[PytestNode], + /, + *default: Any, + include_self: bool = True, +) -> Any: + """Get a parent node in the given scope. + + Use this function to have a correct typing of the returned object, + until ``pytest`` provides a better typing. + + :param node: The node to get an ancestor of. + :param cond: The ancestor type or scope. + :param default: A default value. + :param include_self: Include *node* if possible. + :return: A node in the ancestor chain (possibly *node*) of desired type. + """ + if isinstance(cond, str): + cond = get_node_type_by_scope(cond) + + parent = node.getparent(cond) + if parent is None or parent is node and not include_self: + if default: + return default[0] + msg = f'no parent of type {cond} for {node}' + raise AttributeError(msg) + return parent + + +TestNodeLocation = tuple[str, int] +"""The location ``(fspath, lineno)`` of a pytest node. + +* The *fspath* is relative to :attr:`pytest.Config.rootpath`. +* The line number is a 0-based integer. +""" + + +def get_node_location(node: PytestNode) -> TestNodeLocation | None: + """The node location ``(fspath, lineno)``, if any. + + If the path or the line number cannot be deduced, a warning is emitted. + + When deduced, the line number is a 0-based integer. + """ + path, lineno = get_fslocation_from_item(node) + if not (path := os.fsdecode(path)) or lineno == -1 or lineno is None: + msg = f'could not obtain node location for {node!r}' + warnings.warn_explicit(msg, category=NodeWarning, filename=path, lineno=-1) + return None + return path, lineno + + +def get_mark_parameters( + node: PytestNode, + marker: str, + /, + *default_args: Any, + **default_kwargs: Any, +) -> tuple[list[Any], dict[str, Any]]: + """Get the positional and keyword arguments of node. + + :param node: The pytest node to analyze. + :param marker: The name of the marker to extract the parameters of. + :param default_args: Optional default positional arguments. + :param default_kwargs: Optional default keyword arguments. + :return: The positional and keyword arguments. + + By convention, arguments are not stacked and are collected in + the *reverse* order the marker decorators are specified, e.g.:: + + @pytest.mark.foo('ignored', 2, a='ignored', b=2) + @pytest.mark.foo(1, a=1) + def test(request): + args, kwargs = get_mark_parameters(request.node, 'foo') + assert args == [1, 2] + assert kwargs == {'a': 1, 'b': 2} + """ + args, kwargs = list(default_args), default_kwargs + for info in reversed(list(node.iter_markers(marker))): + args[:len(info.args)] = info.args # fmt: skip + kwargs |= info.kwargs + return args, kwargs + + +def check_mark_keywords( + mark: str, + expect: Collection[str], + actual: Iterable[str], + *, + node: PytestNode | None = None, + ignore_private: bool = False, + strict: bool = False, +) -> bool: + """Check the keyword arguments. + + :param mark: The name of the marker being checked. + :param expect: The marker expected keyword parameter names. + :param actual: The keyword arguments to check. + :param node: Optional node to emit warnings upon invalid arguments. + :param ignore_private: Ignore keyword arguments with leading underscores. + :param strict: If true, raises an exception instead of a warning. + :return: Indicate if the keyword arguments were recognized or not. + + >>> check_mark_keywords('_', ['a', 'b'], {'a': 1, 'b': 2, 'c': 3}) + False + >>> check_mark_keywords('_', ['a', 'b'], {'a': 1, 'b': 2, '_private': 3}, + ... ignore_private=True) + True + """ + keys = set(actual).difference(expect) + extras = sorted(key for key in keys if not (key.startswith('_') and ignore_private)) + if extras and node: + msg = 'unexpected keyword argument(s): %s' % ', '.join(sorted(extras)) + if strict: + pytest.fail(format_mark_failure(mark, msg)) + + issue_warning(node, MarkWarning(msg, mark)) + return False + return len(extras) == 0 + + +def check_mark_str_args(mark: str, /, **kwargs: Any) -> None: + """Check that marker string arguments are either None or non-empty. + + :param mark: The marker name. + :param kwargs: A mapping of marker argument names and their values. + :raise pytest.Failed: The validation failed. + """ + for argname, value in kwargs.items(): + if value and not isinstance(value, str) or not value and value is not None: + fmt = 'expecting a non-empty string or None for %r, got: %r' + pytest.fail(format_mark_failure(mark, fmt % (argname, value))) + + +def stack_pytest_markers( + marker: pytest.MarkDecorator, /, *markers: pytest.MarkDecorator +) -> Callable[[Callable[..., None]], Callable[..., None]]: + """Create a decorator stacking pytest markers.""" + stack = [marker, *markers] + stack.reverse() + + def wrapper(func: Callable[..., None]) -> Callable[..., None]: + for marker in stack: + func = marker(func) + return func + + return wrapper + + +@contextmanager +def pytest_not_raises(*exceptions: type[BaseException]) -> Iterator[None]: + """Context manager asserting that no exception is raised.""" + try: + yield + except exceptions as exc: + pytest.fail(f'DID RAISE {exc.__class__}') + + +# fmt: off +@overload +def issue_warning(config: pytest.Config, warning: Warning, /) -> None: ... # NoQA: E704 +@overload +def issue_warning(config: pytest.Config, fmt: Any, /, *args: Any, category: type[Warning] | None = ...) -> None: ... # NoQA: E501, E704 +@overload +def issue_warning(request: pytest.FixtureRequest, warning: Warning, /) -> None: ... # NoQA: E501, E704 +@overload +def issue_warning(request: pytest.FixtureRequest, fmt: Any, /, *args: Any, category: type[Warning] | None = ...) -> None: ... # NoQA: E501, E704 +@overload +def issue_warning(node: PytestNode, warning: Warning, /) -> None: ... # NoQA: E704 +@overload +def issue_warning(node: PytestNode, fmt: Any, /, *args: Any, category: type[Warning] | None = ...) -> None: ... # NoQA: E501, E704 +# fmt: on +def issue_warning( # NoQA: E302 + ctx: Any, fmt: Any, /, *args: Any, category: type[Warning] | None = None +) -> None: + """Public helper for emitting a warning on a pytest object. + + This is typically useful for debugging when plugins capturing ``print`` + such as ``xdist`` are active. Warnings are (apparently) always printed + on the console. + """ + if isinstance(fmt, Warning): + warning = fmt + else: + message = str(fmt) + if args: # allow str(fmt) to contain '%s' + message = message % args + warning = SphinxTestingWarning(message) if category is None else category(message) + + if isinstance(ctx, pytest.Config): + ctx.issue_config_time_warning(warning, stacklevel=2) + return + + node = ctx.node if isinstance(ctx, pytest.FixtureRequest) else ctx + if not isinstance(node, PytestNode): + err = f'expecting a session, a fixture request or a pytest node, got {node!r}' + raise TypeError(err) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore', NodeWarning) + location = get_node_location(node) + + if location is None: + filename = os.fsdecode(node.path or 'unknown location') + lineno = -1 + else: + filename, lineno = location + lineno = lineno + 1 + + warnings.warn_explicit(warning, category=None, filename=filename, lineno=lineno) + + +def format_mark_failure(mark: str, message: str) -> str: + return f'pytest.mark.{mark}(): {message}' + + +############################################################################### +# _pytest.config.Config accessor +############################################################################### + + +def get_pytest_config( + subject: pytest.Config | pytest.FixtureRequest | PytestNode, / +) -> pytest.Config: + """Get the underlying pytest configuration of the *subject*.""" + if isinstance(subject, pytest.Config): + return subject + + config = getattr(subject, 'config', None) + if config is None or not isinstance(config, pytest.Config): + msg = f'no configuration accessor for {type(subject)} objects' + raise TypeError(msg) + return config + + +############################################################################### +# _pytest.tempdir.TempPathFactory accessor +############################################################################### + +_DT = TypeVar('_DT') + + +# fmt: off +@overload +def get_tmp_path_factory(subject: Any, /) -> pytest.TempPathFactory: ... # NoQA: E704 +@overload +def get_tmp_path_factory(subject: Any, default: _DT, /) -> pytest.TempPathFactory | _DT: ... # NoQA: E501, E704 +# fmt: on +def get_tmp_path_factory(subject: Any, /, *default: Any) -> Any: # NoQA: E302 + """Get the optional underlying path factory of the *subject*.""" + config = get_pytest_config(subject) + factory = getattr(config, '_tmp_path_factory', None) + if factory is None: + if default: + return default[0] + + msg = f'cannot extract the underlying temporary path factory from {subject!r}' + raise AttributeError(msg) + assert isinstance(factory, pytest.TempPathFactory) + return factory diff --git a/sphinx/testing/_internal/pytest_xdist.py b/sphinx/testing/_internal/pytest_xdist.py new file mode 100644 index 00000000000..ab7c7f800d0 --- /dev/null +++ b/sphinx/testing/_internal/pytest_xdist.py @@ -0,0 +1,74 @@ +"""Private utilities for the `pytest-xdist`_ plugin. + +.. _pytest-xdist: https://pytest-xdist.readthedocs.io + +All functions in this module have an undefined behaviour if they are +called before the ``pytest_cmdline_main`` hook. +""" + +from __future__ import annotations + +__all__ = () + +from typing import TYPE_CHECKING, Literal + +if TYPE_CHECKING: + import pytest + +#: Scheduling policy for :mod:`xdist` specified by :option:`!--dist`. +Policy = Literal['no', 'each', 'load', 'loadscope', 'loadfile', 'loadgroup', 'worksteal'] + + +def get_xdist_policy(config: pytest.Config) -> Policy: + """Get the ``config.option.dist`` value even if :mod:`!xdist` is absent. + + Use ``get_xdist_policy(config) != 'no'`` to determine whether the plugin + is active and loaded or not. + """ + # On systems without the :mod:`!xdist` module, the ``dist`` option does + # not even exist in the first place and thus using ``config.option.dist`` + # would raise an :exc:`AttributeError`. + if config.pluginmanager.has_plugin('xdist'): + # worker nodes do not inherit the 'config.option.dist' value + # when used by pytester, but since we have a hook that adds + # them as a worker input, we can retrieve it correctly even + # if we are not in the controller node + if hasattr(config, 'workerinput'): + # this requires our custom xdist hooks + return config.workerinput['sphinx_xdist_policy'] + return config.option.dist + return 'no' + + +def is_pytest_xdist_enabled(config: pytest.Config) -> bool: + """Check that the :mod:`!xdist` plugin is loaded and active. + + :param config: A pytest configuration object. + """ + return get_xdist_policy(config) != 'no' + + +def is_pytest_xdist_controller(config: pytest.Config) -> bool: + """Check if the configuration is attached to the xdist controller. + + If the :mod:`!xdist` plugin is not active, this returns ``False``. + + .. important:: + + This function differs from :func:`xdist.is_xdist_worker` in the + sense that it works even if the :mod:`xdist` plugin is inactive. + """ + return is_pytest_xdist_enabled(config) and not is_pytest_xdist_worker(config) + + +def is_pytest_xdist_worker(config: pytest.Config) -> bool: + """Check if the configuration is attached to a xdist worker. + + If the :mod:`!xdist` plugin is not active, this returns ``False``. + + .. important:: + + This function differs from :func:`xdist.is_xdist_controller` in the + sense that it works even if the :mod:`xdist` plugin is inactive. + """ + return is_pytest_xdist_enabled(config) and hasattr(config, 'workerinput') diff --git a/sphinx/testing/_internal/util.py b/sphinx/testing/_internal/util.py new file mode 100644 index 00000000000..d2025ff0f89 --- /dev/null +++ b/sphinx/testing/_internal/util.py @@ -0,0 +1,117 @@ +"""Private utililty functions for :mod:`sphinx.testing.plugin`. + +This module is an implementation detail and any provided function +or class can be altered, removed or moved without prior notice. +""" + +from __future__ import annotations + +__all__ = () + +import binascii +import json +import os +import pickle +from functools import lru_cache +from typing import TYPE_CHECKING, overload + +import pytest + +if TYPE_CHECKING: + from typing import Any, Final + + from _pytest.nodes import Node as PytestNode + + +UID_BITLEN: int = 32 +r"""The bit-length of unique identifiers generated by this module. + +Must be a power of two in :math:`[8, 128]`. +""" +UID_BUFLEN: Final[int] = UID_BITLEN // 8 +UID_HEXLEN: Final[int] = UID_BITLEN // 4 + + +# fmt: off +@overload +def make_unique_id() -> str: ... # NoQA: E704 +@overload +def make_unique_id(prefix: str | os.PathLike[str]) -> str: ... # NoQA: E704 +# fmt: on +def make_unique_id(prefix: str | os.PathLike[str] | None = None) -> str: # NoQA: E302 + r"""Generate a random unique identifier prefixed by *prefix*. + + :param prefix: An optional prefix to prepend to the unique identifier. + :return: A random unique identifier. + """ + suffix = os.urandom(UID_BUFLEN).hex() + return '-'.join((os.fsdecode(prefix), suffix)) if prefix else suffix + + +def get_objects_checksum(*args: Any, **kwargs: Any) -> int: + """Compute a CRC-32 checksum of arbitrary objects. + + The order of the positional arguments and keyword arguments matters + when computing the checksum, hence it is recommended to only use + keyword arguments whenever possible. + + If an object cannot be pickled, its representation is based on the value + of its :func:`id`, possibly making the checksum distinct for equivalent + but non-pickable objects. + """ + + def default_encoder(x: object) -> str: + try: + return pickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL).hex() + except (NotImplementedError, TypeError, ValueError): + return format(id(x), 'x') + + # use the most compact JSON format + data = [args, kwargs] + env = json.dumps(data, separators=(',', ':'), default=default_encoder, sort_keys=True) + # avoid using unique_object_id() since we do not really need SHA-1 entropy + return binascii.crc32(env.encode('utf-8')) + + +@lru_cache(maxsize=65536) +def unique_object_id(name: str) -> str: + """Get a unique hexadecimal identifier for an object name. + + :param name: The name of the object to get a unique ID of. + :return: A unique hexadecimal identifier for *name*. + """ + from hashlib import sha1 + + # ensure that non UTF-8 characters are supported and handled similarly + h = sha1(name.encode('utf-8', errors='backslashreplace')) + byt = int.from_bytes(h.digest()[:UID_BUFLEN], byteorder='little') + return f'%00{UID_HEXLEN}x' % byt + + +def get_container_id(node: PytestNode) -> str: + """Get a unique identifier for the node's container. + + The node's container is defined by all but the last component of the + node's path (e.g., ``pkg.mod.test_func`` is contained in ``pkg.mod``). + """ + + def get_obj_name(subject: PytestNode) -> str | None: + if isinstance(subject, pytest.Package): + return subject.name + if isinstance(subject, (pytest.Module, pytest.Class)): + return getattr(subject.obj, '__name__', None) + return None + + names = map(get_obj_name, node.listchain()) + container = '@'.join(filter(None, names)) or node.nodeid + return unique_object_id(container) + + +def get_location_id(location: tuple[str, int]) -> str: + """Get a unique hexadecimal identifier out of a test location. + + The ID is based on the physical node location (file and line number). + The line number is a 0-based integer but can be -1 if unknown. + """ + fspath, lineno = location + return unique_object_id(f'{fspath}:L{lineno}') diff --git a/sphinx/testing/_internal/warnings.py b/sphinx/testing/_internal/warnings.py new file mode 100644 index 00000000000..225c56e45b6 --- /dev/null +++ b/sphinx/testing/_internal/warnings.py @@ -0,0 +1,31 @@ +"""Warnings emitted by the :mod:`sphinx.testing.plugin` plugin.""" + +from __future__ import annotations + +__all__ = () + +from _pytest.warning_types import PytestWarning + + +class SphinxTestingWarning(PytestWarning): + """Base class for warnings emitted during test configuration.""" + + +class NodeWarning(SphinxTestingWarning): + """A warning emitted when an operation on a pytest node failed.""" + + +class MarkWarning(NodeWarning): + """A warning emitted when parsing a marker.""" + + def __init__(self, message: str, markname: str | None = None) -> None: + message = f'@pytest.mark.{markname}(): {message}' if markname else message + super().__init__(message) + + +class FixtureWarning(NodeWarning): + """A warning emitted during a fixture configuration.""" + + def __init__(self, message: str, fixturename: str | None = None) -> None: + message = f'FIXTURE({fixturename!r}): {message}' if fixturename else message + super().__init__(message) diff --git a/sphinx/testing/_xdist_hooks.py b/sphinx/testing/_xdist_hooks.py new file mode 100644 index 00000000000..9be49d8b0b2 --- /dev/null +++ b/sphinx/testing/_xdist_hooks.py @@ -0,0 +1,40 @@ +"""Hooks to register when the ``xdist`` plugin is active. + +Wen ``xdist`` is active, the controller node automatically loads +this module through :func:`sphinx.testing.plugin.pytest_addhooks`. +""" + +from __future__ import annotations + +__all__ = () + +import shutil +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pytest + from execnet import XSpec + from xdist.workermanage import NodeManager, WorkerController + + +def pytest_xdist_setupnodes(config: pytest.Config, specs: list[XSpec]) -> None: + """Setup the environment of each worker controller node.""" + columns = shutil.get_terminal_size()[0] + for spec in specs: + # ensure that the controller nodes inherit the same terminal width + spec.env.setdefault('COLUMNS', str(columns)) + + +def pytest_configure_node(node: WorkerController) -> None: + node_config: pytest.Config = node.config + # the node's config is not the same as the controller's config + assert node_config.pluginmanager.has_plugin('xdist'), 'xdist is not loaded' + + manager: NodeManager = node.nodemanager + config: pytest.Config = manager.config + assert config.pluginmanager.has_plugin('xdist'), 'xdist is not loaded' + + # worker nodes do not inherit the 'config.option.dist' value + # when used by pytester, so we simply copy it from the main + # controller to the worker node + node.workerinput['sphinx_xdist_policy'] = config.option.dist diff --git a/sphinx/testing/fixtures.py b/sphinx/testing/fixtures.py index 6e1a1222a5f..14cf58451eb 100644 --- a/sphinx/testing/fixtures.py +++ b/sphinx/testing/fixtures.py @@ -2,224 +2,505 @@ from __future__ import annotations +import contextlib +import os import shutil import subprocess import sys -from collections import namedtuple -from io import StringIO -from typing import TYPE_CHECKING +import warnings +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, cast import pytest -from sphinx.testing.util import SphinxTestApp, SphinxTestAppWrapperForSkipBuilding +from sphinx.deprecation import RemovedInSphinx90Warning +from sphinx.testing._internal.cache import AppInfo, LegacyModuleCache, ModuleCache +from sphinx.testing._internal.isolation import Isolation +from sphinx.testing._internal.markers import ( + AppLegacyParams, + AppParams, + process_isolate, + process_sphinx, + process_test_params, +) +from sphinx.testing._internal.pytest_util import ( + TestRootFinder, + find_context, + get_mark_parameters, +) +from sphinx.testing._internal.pytest_xdist import is_pytest_xdist_enabled +from sphinx.testing.util import ( + SphinxTestApp, + SphinxTestAppLazyBuild, + SphinxTestAppWrapperForSkipBuilding, +) +from sphinx.util.console import strip_escape_sequences if TYPE_CHECKING: - from collections.abc import Callable, Iterator + from collections.abc import Iterator + from io import StringIO from pathlib import Path - from typing import Any + from typing import Any, Final, Union -DEFAULT_ENABLED_MARKERS = [ + from _pytest.nodes import Node as PytestNode + + from sphinx.testing._internal.isolation import IsolationPolicy + from sphinx.testing._internal.markers import TestParams + + AnySphinxTestApp = Union[SphinxTestApp, SphinxTestAppWrapperForSkipBuilding] + AnyAppParams = Union[AppParams, AppLegacyParams] + +DEFAULT_ENABLED_MARKERS: Final[list[str]] = [ # The marker signature differs from the constructor signature # since the way it is processed assumes keyword arguments for - # the 'testroot' and 'srcdir'. + # the 'testroot' and 'srcdir'. In addition, 'freshenv' and + # 'isolate' are mutually exclusive arguments (and the latter + # is recommended over the former). ( 'sphinx(' 'buildername="html", *, ' 'testroot="root", srcdir=None, ' - 'confoverrides=None, freshenv=False, ' + 'confoverrides=None, freshenv=None, ' 'warningiserror=False, tags=None, verbosity=0, parallel=0, ' - 'keep_going=False, builddir=None, docutils_conf=None' + 'keep_going=False, builddir=None, docutils_conf=None, ' + 'isolate=False' '): arguments to initialize the sphinx test application.' ), - 'test_params(shared_result=...): test parameters.', + 'test_params(*, shared_result=None): test configuration.', + 'isolate(policy=None, /): test isolation policy.', + 'sphinx_no_default_xdist(): disable the default xdist-group on tests', ] +############################################################################### +# pytest hooks +# +# *** IMPORTANT *** +# +# The hooks must be compatible with the legacy plugin until Sphinx 9.x. +############################################################################### + + +def pytest_addhooks(pluginmanager: pytest.PytestPluginManager) -> None: + if pluginmanager.has_plugin('xdist'): + from sphinx.testing import _xdist_hooks + + # the legacy plugin does not really care about this plugin + # since it only depends on 'xdist' and not on sphinx itself + pluginmanager.register(_xdist_hooks, name='sphinx-xdist-hooks') + + def pytest_configure(config: pytest.Config) -> None: - """Register custom markers""" + """Register custom markers.""" for marker in DEFAULT_ENABLED_MARKERS: config.addinivalue_line('markers', marker) -@pytest.fixture(scope='session') -def rootdir() -> str | None: - return None +@pytest.hookimpl() +def pytest_runtest_makereport( + item: pytest.Item, call: pytest.CallInfo +) -> pytest.TestReport | None: + if call.when != 'teardown' or _APP_INFO_KEY not in item.stash: + return None + # handle the delayed test report when using xdist + info = item.stash[_APP_INFO_KEY] + text = _cleanup_app_info(info.render(nodeid=item.nodeid)) + item.add_report_section(call.when, 'fixture: %r' % 'app', text) + return pytest.TestReport.from_item_and_call(item, call) -class SharedResult: - cache: dict[str, dict[str, str]] = {} - def store(self, key: str, app_: SphinxTestApp) -> Any: - if key in self.cache: - return - data = { - 'status': app_.status.getvalue(), - 'warning': app_.warning.getvalue(), - } - self.cache[key] = data - - def restore(self, key: str) -> dict[str, StringIO]: - if key not in self.cache: - return {} - data = self.cache[key] - return { - 'status': StringIO(data['status']), - 'warning': StringIO(data['warning']), - } +############################################################################### +# sphinx fixtures +############################################################################### @pytest.fixture() -def app_params( - request: Any, - test_params: dict, - shared_result: SharedResult, - sphinx_test_tempdir: str, - rootdir: str, -) -> _app_params: - """ - Parameters that are specified by 'pytest.mark.sphinx' for - sphinx.application.Sphinx initialization - """ - # ##### process pytest.mark.sphinx +def sphinx_use_legacy_plugin() -> bool: # xref RemovedInSphinx90Warning + """If true, use the legacy implementation of fixtures. - pargs: dict[int, Any] = {} - kwargs: dict[str, Any] = {} + Redefine this fixture in ``conftest.py`` or at the test level to use + the new plugin implementation (note that the test code might require + changes). By default, the new implementation is disabled so that no + breaking changes occur outside of Sphinx itself. + """ + return True - # to avoid stacking positional args - for info in reversed(list(request.node.iter_markers("sphinx"))): - pargs |= dict(enumerate(info.args)) - kwargs.update(info.kwargs) - args = [pargs[i] for i in sorted(pargs.keys())] +@pytest.fixture(scope='session') +def sphinx_test_tempdir(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Fixture for a temporary directory.""" + return tmp_path_factory.getbasetemp() - # ##### process pytest.mark.test_params - if test_params['shared_result']: - if 'srcdir' in kwargs: - msg = 'You can not specify shared_result and srcdir in same time.' - pytest.fail(msg) - kwargs['srcdir'] = test_params['shared_result'] - restore = shared_result.restore(test_params['shared_result']) - kwargs.update(restore) - # ##### prepare Application params +@pytest.fixture() +def sphinx_builder(request: pytest.FixtureRequest) -> str: + """Fixture for the default builder name.""" + return getattr(request, 'param', 'html') - testroot = kwargs.pop('testroot', 'root') - kwargs['srcdir'] = srcdir = sphinx_test_tempdir / kwargs.get('srcdir', testroot) - # special support for sphinx/tests - if rootdir and not srcdir.exists(): - testroot_path = rootdir / ('test-' + testroot) - shutil.copytree(testroot_path, srcdir) +@pytest.fixture() +def sphinx_isolation() -> IsolationPolicy: + """Fixture for the default isolation policy. - return _app_params(args, kwargs) + This fixture is ignored when using the legacy plugin. + """ + return False -_app_params = namedtuple('_app_params', 'args,kwargs') +@pytest.fixture() +def rootdir() -> str | os.PathLike[str] | None: + """Fixture for the directory containing the testroot directories.""" + return None @pytest.fixture() -def test_params(request: Any) -> dict: +def testroot_prefix() -> str | None: + """Fixture for the testroot directories prefix. + + This fixture is ignored when using the legacy plugin. """ - Test parameters that are specified by 'pytest.mark.test_params' + return 'test-' + + +@pytest.fixture() +def default_testroot() -> str | None: + """Dynamic fixture for the default testroot ID. - :param Union[str] shared_result: - If the value is provided, app._status and app._warning objects will be - shared in the parametrized test functions and/or test functions that - have same 'shared_result' value. - **NOTE**: You can not specify both shared_result and srcdir. + This fixture is ignored when using the legacy plugin. """ - env = request.node.get_closest_marker('test_params') - kwargs = env.kwargs if env else {} - result = { - 'shared_result': None, - } - result.update(kwargs) + return 'root' - if result['shared_result'] and not isinstance(result['shared_result'], str): - msg = 'You can only provide a string type of value for "shared_result"' - raise pytest.Exception(msg) - return result + +@pytest.fixture() +def testroot_finder( + rootdir: str | os.PathLike[str] | None, + testroot_prefix: str | None, + default_testroot: str | None, +) -> TestRootFinder: + """Fixture for the testroot finder object.""" + return TestRootFinder(rootdir, testroot_prefix, default_testroot) + + +############################################################################### +# fixture: app_params() +############################################################################### + + +def _init_sources(src: str | None, dst: Path, isolation: Isolation) -> None: + if src is None or dst.exists(): + return + + if not os.path.exists(src): + pytest.fail(f'no sources found at: {src!r}') + + # make a copy of the testroot + shutil.copytree(src, dst) + + # make the files read-only if isolation is not specified + # to protect the tests against some side-effects (not all + # side-effects will be prevented) + if isolation is Isolation.minimal: + for dirpath, _, filenames in os.walk(dst): + for filename in filenames: + os.chmod(os.path.join(dirpath, filename), 0o444) + + +def __app_params_fixture( + request: pytest.FixtureRequest, + test_params: TestParams, + module_cache: ModuleCache, + sphinx_test_tempdir: Path, + sphinx_builder: str, + sphinx_isolation: IsolationPolicy, + testroot_finder: TestRootFinder, +) -> AppParams: + default_isolation = process_isolate(request.node, sphinx_isolation) + shared_result_id = test_params['shared_result'] + args, kwargs = process_sphinx( + request.node, + session_temp_dir=sphinx_test_tempdir, + testroot_finder=testroot_finder, + default_builder=sphinx_builder, + default_isolation=default_isolation, + shared_result=shared_result_id, + ) + assert shared_result_id == kwargs['shared_result'] + # restore the I/O stream values + if shared_result_id and (frame := module_cache.restore(shared_result_id)): + if kwargs.setdefault('status', frame['status']) is not frame['status']: + fmt = 'cannot use %r when %r is explicitly given' + pytest.fail(fmt % ('shared_result', 'status')) + if kwargs.setdefault('warning', frame['warning']) is not frame['warning']: + fmt = 'cannot use %r when %r is explicitly given' + pytest.fail(fmt % ('shared_result', 'warning')) + + # copy the testroot files to the test sources directory + _init_sources(kwargs['testroot_path'], kwargs['srcdir'], kwargs['isolate']) + return AppParams(args, kwargs) @pytest.fixture() -def app( - test_params: dict, - app_params: tuple[dict, dict], - make_app: Callable, - shared_result: SharedResult, -) -> Iterator[SphinxTestApp]: - """ - Provides the 'sphinx.application.Sphinx' object +def app_params( + request: pytest.FixtureRequest, + test_params: TestParams, + module_cache: ModuleCache, + shared_result: LegacyModuleCache, # xref RemovedInSphinx90Warning + sphinx_test_tempdir: Path, + sphinx_builder: str, + sphinx_isolation: IsolationPolicy, + testroot_finder: TestRootFinder, + sphinx_use_legacy_plugin: bool, # xref RemovedInSphinx90Warning +) -> AppParams | AppLegacyParams: + """Parameters that are specified by ``pytest.mark.sphinx``. + + See :class:`sphinx.testing.util.SphinxTestApp` for the allowed parameters. """ - args, kwargs = app_params - app_ = make_app(*args, **kwargs) - yield app_ + if sphinx_use_legacy_plugin: + msg = ('legacy implementation of sphinx.testing.fixtures is ' + 'deprecated; consider redefining sphinx_use_legacy_plugin() ' + 'in conftest.py to return False.') + warnings.warn(msg, RemovedInSphinx90Warning, stacklevel=2) + return __app_params_fixture_legacy( + request, test_params, shared_result, + sphinx_test_tempdir, testroot_finder.path, + ) - print('# testroot:', kwargs.get('testroot', 'root')) - print('# builder:', app_.builder.name) - print('# srcdir:', app_.srcdir) - print('# outdir:', app_.outdir) - print('# status:', '\n' + app_._status.getvalue()) - print('# warning:', '\n' + app_._warning.getvalue()) + return __app_params_fixture( + request, test_params, module_cache, + sphinx_test_tempdir, sphinx_builder, + sphinx_isolation, testroot_finder, + ) - if test_params['shared_result']: - shared_result.store(test_params['shared_result'], app_) + +############################################################################### +# fixture: test_params() +############################################################################### @pytest.fixture() -def status(app: SphinxTestApp) -> StringIO: +def test_params(request: pytest.FixtureRequest) -> TestParams: + """Test parameters that are specified by ``pytest.mark.test_params``. + + This ``pytest.mark.test_params`` marker takes an optional keyword argument, + namely the *shared_result*, which is a string, e.g.:: + + def test_no_shared_result(test_params): + assert test_params['shared_result'] is None + + @pytest.mark.test_params() + def test_with_random_shared_result(test_params): + assert test_params['shared_result'] == 'some-random-string' + + @pytest.mark.test_params(shared_result='foo') + def test_with_explicit_shared_result(test_params): + assert test_params['shared_result'] == 'foo' + + If the *shared_result* is provided, the ``app.status`` and ``app.warning`` + objects will be shared in the test functions, possibly parametrized, that + have the same *shared_result* value. + + .. note:: + + The *srcdir* parameter of the ``@pytest.mark.sphinx()`` marker and + the *shared_result* parameter of the ``@pytest.mark.test_params()`` + marker are mutually exclusive. """ - Back-compatibility for testing with previous @with_app decorator + return process_test_params(request.node) + + +############################################################################### +# fixture: app() +############################################################################### + + +_APP_INFO_KEY: pytest.StashKey[AppInfo] = pytest.StashKey() + + +def _cleanup_app_info(text: str) -> str: + if os.name == 'nt': + text = strip_escape_sequences(text) + text = text.encode('ascii', errors='backslashreplace').decode('ascii') + return text + + +@contextlib.contextmanager +def _app_info_context( + node: PytestNode, + app: SphinxTestApp, + app_params: AppParams, +) -> Iterator[None]: + # create or get the current :class:`AppInfo` object of the node + if _APP_INFO_KEY not in node.stash: + node.stash[_APP_INFO_KEY] = AppInfo( + builder=app.builder.name, + testroot_path=app_params.kwargs['testroot_path'], + shared_result=app_params.kwargs['shared_result'], + srcdir=os.fsdecode(app.srcdir), + outdir=os.fsdecode(app.outdir), + ) + + app_info = node.stash[_APP_INFO_KEY] + yield + app_info.update(app) + + if not is_pytest_xdist_enabled(node.config) and node.config.option.capture == 'no': + # With xdist, we will print at the test the information but only + # if it is being used with '-s', which has no effect when used by + # xdist since the latter does not support capturing. + # + # + # In addition, use CRLF to avoid being printed inside the + # progression bar (note that we need to render it here so + # that the terminal width is correctly determined). + text = app_info.render(nodeid=node.nodeid) + print('\n', _cleanup_app_info(text), sep='', end='') # NoQA: T201 + + +@pytest.fixture() +def app_info_extras( + request: pytest.FixtureRequest, + # ``app`` is not used but is marked as a dependency so that + # the AppInfo() object is automatically created for *app* + app: AnySphinxTestApp, # xref RemovedInSphinx90Warning: update type + sphinx_use_legacy_plugin: bool, # xref RemovedInSphinx90Warning +) -> dict[str, Any]: + """Fixture to update the information to render at the end of a test. + + Use this fixture in a ``conftest.py`` file or in a test file as follows:: + + @pytest.fixture(autouse=True) + def _add_app_info_extras(app, app_info_extras): + app_info_extras.update(my_extra=1234) + app_info_extras.update(app_extras=app.extras) + + .. note:: + + This fixture is only available if :func:`sphinx_use_legacy_plugin` is + configured to return ``False`` (i.e., the legacy plugin is disabled). """ + # xref RemovedInSphinx90Warning: remove the assert + assert not sphinx_use_legacy_plugin, 'legacy plugin does not support this fixture' + assert _APP_INFO_KEY in request.node.stash + return request.node.stash[_APP_INFO_KEY].extras + + +def __app_fixture( + request: pytest.FixtureRequest, + app_params: AppParams, + make_app: Callable[..., SphinxTestApp], + module_cache: ModuleCache, +) -> Iterator[SphinxTestApp]: + shared_result = app_params.kwargs['shared_result'] + + app = make_app(*app_params.args, **app_params.kwargs) + with _app_info_context(request.node, app, app_params): + yield app + + if shared_result is not None: + module_cache.store(shared_result, app) + + +@pytest.fixture() +def app( + request: pytest.FixtureRequest, + app_params: AnyAppParams, # xref RemovedInSphinx90Warning: update type + test_params: TestParams, # xref RemovedInSphinx90Warning + make_app: Callable[..., AnySphinxTestApp], # xref RemovedInSphinx90Warning: update type + module_cache: ModuleCache, + shared_result: LegacyModuleCache, # xref RemovedInSphinx90Warning + sphinx_use_legacy_plugin: bool, # xref RemovedInSphinx90Warning +) -> Iterator[AnySphinxTestApp]: # xref RemovedInSphinx90Warning: update type + """A :class:`sphinx.application.Sphinx` object suitable for testing.""" + if sphinx_use_legacy_plugin: # xref RemovedInSphinx90Warning + # a warning will be emitted by the app_params fixture + app_params = cast(AppLegacyParams, app_params) + fixt = __app_fixture_legacy(request, app_params, test_params, make_app, shared_result) + else: + # xref RemovedInSphinx90Warning: remove the cast + app_params = cast(AppParams, app_params) + make_app = cast(Callable[..., SphinxTestApp], make_app) + fixt = __app_fixture(request, app_params, make_app, module_cache) + + yield from fixt + return + +############################################################################### +# other fixtures +############################################################################### + + +@pytest.fixture() +def status(app: AnySphinxTestApp) -> StringIO: # xref RemovedInSphinx90Warning: narrow type + """Fixture for the :func:`~sphinx.testing.plugin.app` status stream.""" return app.status @pytest.fixture() -def warning(app: SphinxTestApp) -> StringIO: - """ - Back-compatibility for testing with previous @with_app decorator - """ +def warning(app: AnySphinxTestApp) -> StringIO: # xref RemovedInSphinx90Warning: narrow type + """Fixture for the :func:`~sphinx.testing.plugin.app` warning stream.""" return app.warning @pytest.fixture() -def make_app(test_params: dict, monkeypatch: Any) -> Iterator[Callable]: - """ - Provides make_app function to initialize SphinxTestApp instance. - if you want to initialize 'app' in your test function. please use this - instead of using SphinxTestApp class directory. - """ - apps = [] - syspath = sys.path.copy() +def make_app( + test_params: TestParams, + sphinx_use_legacy_plugin: bool, # xref RemovedInSphinx90Warning + # xref RemovedInSphinx90Warning: narrow callable return type +) -> Iterator[Callable[..., AnySphinxTestApp]]: + """Fixture to create :class:`~sphinx.testing.util.SphinxTestApp` objects.""" + stack: list[SphinxTestApp] = [] + allow_rebuild = test_params['shared_result'] is None + + # xref RemovedInSphinx90Warning: narrow return type + def make(*args: Any, **kwargs: Any) -> AnySphinxTestApp: + if allow_rebuild: + app = SphinxTestApp(*args, **kwargs) + else: + if sphinx_use_legacy_plugin: # xref RemovedInSphinx90Warning + subject = SphinxTestApp(*args, **kwargs) + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=RemovedInSphinx90Warning) + app = SphinxTestAppWrapperForSkipBuilding(subject) # type: ignore[assignment] # NoQA: E501 + else: + app = SphinxTestAppLazyBuild(*args, **kwargs) + stack.append(app) + return app - def make(*args: Any, **kwargs: Any) -> SphinxTestApp: - status, warning = StringIO(), StringIO() - kwargs.setdefault('status', status) - kwargs.setdefault('warning', warning) - app_: Any = SphinxTestApp(*args, **kwargs) - apps.append(app_) - if test_params['shared_result']: - app_ = SphinxTestAppWrapperForSkipBuilding(app_) - return app_ + syspath = sys.path.copy() yield make - sys.path[:] = syspath - for app_ in reversed(apps): # clean up applications from the new ones - app_.cleanup() + + while stack: + stack.pop().cleanup() + + +_MODULE_CACHE_STASH_KEY: pytest.StashKey[ModuleCache] = pytest.StashKey() @pytest.fixture() -def shared_result() -> SharedResult: - return SharedResult() +def module_cache(request: pytest.FixtureRequest) -> ModuleCache: + """A :class:`ModuleStorage` object.""" + module = find_context(request.node, 'module') + return module.stash.setdefault(_MODULE_CACHE_STASH_KEY, ModuleCache()) @pytest.fixture(scope='module', autouse=True) -def _shared_result_cache() -> None: - SharedResult.cache.clear() +def _module_cache_clear(request: pytest.FixtureRequest) -> None: + """Cleanup the shared result cache for the test module. + + This fixture is automatically invoked. + """ + module = find_context(request.node, 'module') + cache = module.stash.get(_MODULE_CACHE_STASH_KEY, None) + if cache is not None: + cache.clear() @pytest.fixture() -def if_graphviz_found(app: SphinxTestApp) -> None: # NoQA: PT004 +# xref RemovedInSphinx90Warning: update type +def if_graphviz_found(app: AnySphinxTestApp) -> None: # NoQA: PT004 """ The test will be skipped when using 'if_graphviz_found' fixture and graphviz dot command is not found. @@ -236,10 +517,50 @@ def if_graphviz_found(app: SphinxTestApp) -> None: # NoQA: PT004 pytest.skip('graphviz "dot" is not available') +_HOST_ONLINE_ERROR = pytest.StashKey[Optional[str]]() + + +def _query(address: tuple[str, int]) -> str | None: + import socket + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.settimeout(5) + sock.connect(address) + except OSError as exc: + # other type of errors are propagated + return str(exc) + return None + + @pytest.fixture(scope='session') -def sphinx_test_tempdir(tmp_path_factory: Any) -> Path: - """Temporary directory.""" - return tmp_path_factory.getbasetemp() +def sphinx_remote_query_address() -> tuple[str, int]: + """Address to which a query is made to check that the host is online. + + By default, onlineness is tested by querying the DNS server ``1.1.1.1`` + but users concerned about privacy might change it in ``conftest.py``. + """ + return ('1.1.1.1', 80) + + +@pytest.fixture(scope='session') +def if_online( # NoQA: PT004 + request: pytest.FixtureRequest, + sphinx_remote_query_address: tuple[str, int], +) -> None: + """Skip the test if the host has no connection. + + Usage:: + + @pytest.mark.usefixtures('if_online') + def test_if_host_is_online(): ... + """ + if _HOST_ONLINE_ERROR not in request.session.stash: + # do not use setdefault() to avoid creating a socket connection + lookup_error = _query(sphinx_remote_query_address) + request.session.stash[_HOST_ONLINE_ERROR] = lookup_error + if (error := request.session.stash[_HOST_ONLINE_ERROR]) is not None: + pytest.skip('host appears to be offline (%s)' % error) @pytest.fixture() @@ -251,10 +572,95 @@ def rollback_sysmodules() -> Iterator[None]: # NoQA: PT004 For example, used in test_ext_autosummary.py to permit unloading the target module to clear its cache. """ - sysmodules = list(sys.modules) + sysmodules = frozenset(sys.modules) try: yield finally: for modname in list(sys.modules): if modname not in sysmodules: sys.modules.pop(modname) + + +############################################################################### +# sphinx deprecated fixtures +# +# Once we are in version 9.x, we can remove the private implementations +# and clean-up the fixtures so that they use a single implementation. +############################################################################### + + +def __app_params_fixture_legacy( # xref RemovedInSphinx90Warning + request: pytest.FixtureRequest, + test_params: TestParams, + shared_result: LegacyModuleCache, + sphinx_test_tempdir: Path, + rootdir: str | os.PathLike[str] | None, +) -> AppLegacyParams: + """ + Parameters that are specified by 'pytest.mark.sphinx' for + sphinx.application.Sphinx initialization + """ + # ##### process pytest.mark.sphinx + args, kwargs = get_mark_parameters(request.node, 'sphinx') + + # ##### process pytest.mark.test_params + if test_params['shared_result']: + if 'srcdir' in kwargs: + msg = 'You can not specify shared_result and srcdir in same time.' + pytest.fail(msg) + kwargs['srcdir'] = test_params['shared_result'] + restore = shared_result.restore(test_params['shared_result']) + kwargs.update(restore) + + testroot = kwargs.pop('testroot', 'root') + kwargs['srcdir'] = srcdir = sphinx_test_tempdir / kwargs.get('srcdir', testroot) + + # special support for sphinx/tests + if rootdir and not srcdir.exists(): + testroot_path = os.path.join(rootdir, 'test-' + testroot) + shutil.copytree(testroot_path, srcdir) + + return AppLegacyParams(args, kwargs) + + +def __app_fixture_legacy( # xref RemovedInSphinx90Warning + request: pytest.FixtureRequest, + app_params: AppLegacyParams, + test_params: TestParams, + make_app: Callable[..., AnySphinxTestApp], + shared_result: LegacyModuleCache, +) -> Iterator[AnySphinxTestApp]: + app = make_app(*app_params.args, **app_params.kwargs) + yield app + + print('# testroot:', app_params.kwargs.get('testroot', 'root')) + print('# builder:', app.builder.name) + print('# srcdir:', app.srcdir) + print('# outdir:', app.outdir) + print('# status:', '\n' + app.status.getvalue()) + print('# warning:', '\n' + app.warning.getvalue()) + + if test_params['shared_result']: + shared_result.store(test_params['shared_result'], app) + + +@pytest.fixture() +def shared_result( # xref RemovedInSphinx90Warning + request: pytest.FixtureRequest, + sphinx_use_legacy_plugin: bool, +) -> LegacyModuleCache: + if ( + not {'app', 'app_params'}.intersection(request.fixturenames) + and not sphinx_use_legacy_plugin + ): + # warn a direct usage of this fixture + warnings.warn("this fixture is deprecated", RemovedInSphinx90Warning, stacklevel=2) + return LegacyModuleCache() + + +@pytest.fixture(scope='module', autouse=True) +def _shared_result_cache() -> None: # xref RemovedInSphinx90Warning + LegacyModuleCache.cache.clear() + + +SharedResult = LegacyModuleCache # xref RemovedInSphinx90Warning diff --git a/sphinx/testing/util.py b/sphinx/testing/util.py index b2df709eea8..9f86e1ff75f 100644 --- a/sphinx/testing/util.py +++ b/sphinx/testing/util.py @@ -7,6 +7,7 @@ import contextlib import os import sys +import warnings from io import StringIO from types import MappingProxyType from typing import TYPE_CHECKING @@ -17,6 +18,7 @@ import sphinx.application import sphinx.locale import sphinx.pycode +from sphinx.deprecation import RemovedInSphinx90Warning from sphinx.util.console import strip_colors from sphinx.util.docutils import additional_nodes @@ -28,6 +30,8 @@ from docutils.nodes import Node + from sphinx.environment import BuildEnvironment + def assert_node(node: Node, cls: Any = None, xpath: str = "", **kwargs: Any) -> None: if cls: @@ -205,14 +209,65 @@ def build(self, force_all: bool = False, filenames: list[str] | None = None) -> super().build(force_all, filenames) +class SphinxTestAppLazyBuild(SphinxTestApp): + """Class to speed-up tests with common resources. + + This class is used to speed up the test by skipping ``app.build()`` if + it has already been built and there are any output files. + + Note that it is incorrect to use ``app.build(force_all=True)`` since + this flag assumes that the sources must be read once again to generate + the output, e.g.:: + + @pytest.mark.sphinx('text', testroot='foo') + @pytest.mark.test_params(shared_result='foo') + def test_foo_project_text1(app): + app.build() + + @pytest.mark.sphinx('text', testroot='foo') + @pytest.mark.test_params(shared_result='foo') + def test_foo_project_text2(app): + # If we execute test_foo_project_text1() before, + # then we should assume that the build phase is + # a no-op. So "force_all" should have no effect. + app.build(force_all=True) # BAD + + Be careful not to use different values for *filenames* in a lazy build + since only the first set of filenames that produce an output would be + considered. + """ + + def _init_env(self, freshenv: bool) -> BuildEnvironment: + if freshenv: + raise ValueError('cannot use %r in lazy builds' % 'freshenv=True') + return super()._init_env(freshenv) + + def build(self, force_all: bool = False, filenames: list[str] | None = None) -> None: + if force_all: + raise ValueError('cannot use %r in lazy builds' % 'force_all=True') + + # see: https://docs.python.org/3/library/os.html#os.scandir + with os.scandir(self.outdir) as it: + has_files = next(it, None) is not None + + if not has_files: # build if no files were already built + super().build(force_all=False, filenames=filenames) + + +# for backward compatibility class SphinxTestAppWrapperForSkipBuilding: - """A wrapper for SphinxTestApp. + """Class to speed-up tests with common resources. - This class is used to speed up the test by skipping ``app.build()`` - if it has already been built and there are any output files. + This class is used to speed up the test by skipping ``app.build()`` if + it has already been built and there are any output files. """ def __init__(self, app_: SphinxTestApp) -> None: + warnings.warn( + f'{self.__class__.__name__!r} is deprecated, use ' + f'{SphinxTestAppLazyBuild.__name__!r} instead', + category=RemovedInSphinx90Warning, stacklevel=2, + ) self.app = app_ def __getattr__(self, name: str) -> Any: diff --git a/tests/conftest.py b/tests/conftest.py index 1c8d5250e80..88660ec9611 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,11 +11,14 @@ import sphinx import sphinx.locale import sphinx.pycode +from sphinx.testing._internal.pytest_util import get_tmp_path_factory from sphinx.testing.util import _clean_up_global_state if TYPE_CHECKING: from collections.abc import Iterator + from _pytest.config import Config + def _init_console( locale_dir: str | None = sphinx.locale._LOCALE_DIR, catalog: str = 'sphinx', @@ -31,24 +34,57 @@ def _init_console( sphinx.locale.init_console = _init_console +# for now, we do not enable the 'xdist' plugin pytest_plugins = ['sphinx.testing.fixtures'] -# Exclude 'roots' dirs for pytest test collector -collect_ignore = ['roots'] +# Exclude resource directories for pytest test collector +collect_ignore = ['certs', 'roots'] os.environ['SPHINX_AUTODOC_RELOAD_MODULES'] = '1' +############################################################################### +# pytest hooks +############################################################################### + + +def pytest_configure(config: Config) -> None: + config.addinivalue_line( + 'markers', + 'apidoc(*, coderoot="test-root", excludes=[], options=[]): ' + 'sphinx-apidoc command-line options (see test_ext_apidoc).', + ) + + +def pytest_report_header(config: Config) -> str: + headers: dict[str, str] = { + 'libraries': f'Sphinx-{sphinx.__display_version__}, docutils-{docutils.__version__}', + } + if (factory := get_tmp_path_factory(config, None)) is not None: + headers['base tmp_path'] = os.fsdecode(factory.getbasetemp()) + return '\n'.join(f'{key}: {value}' for key, value in headers.items()) + + +############################################################################### +# fixtures +############################################################################### + + +@pytest.fixture() +def sphinx_use_legacy_plugin() -> bool: # xref RemovedInSphinx90Warning + return False # use the new implementation + + @pytest.fixture(scope='session') def rootdir() -> Path: return Path(__file__).parent.resolve() / 'roots' -def pytest_report_header(config: pytest.Config) -> str: - header = f"libraries: Sphinx-{sphinx.__display_version__}, docutils-{docutils.__version__}" - if hasattr(config, '_tmp_path_factory'): - header += f"\nbase tmp_path: {config._tmp_path_factory.getbasetemp()}" - return header +# TODO(picnixz): change this fixture to 'minimal' when all tests using 'root' +# have been found and explicitly changed +@pytest.fixture(scope='session') +def default_testroot() -> str: + return 'root' @pytest.fixture(autouse=True) diff --git a/tests/roots/test-minimal/conf.py b/tests/roots/test-minimal/conf.py new file mode 100644 index 00000000000..89250616221 --- /dev/null +++ b/tests/roots/test-minimal/conf.py @@ -0,0 +1,3 @@ +# minimal test root +include_patterns = ['index.rst'] +exclude_patterns = ['_build'] diff --git a/tests/roots/test-minimal/index.rst b/tests/roots/test-minimal/index.rst new file mode 100644 index 00000000000..8260ebead4e --- /dev/null +++ b/tests/roots/test-minimal/index.rst @@ -0,0 +1 @@ +.. empty index \ No newline at end of file diff --git a/tests/test_builders/test_build_html.py b/tests/test_builders/test_build_html.py index 1fa3ba4cd13..0d4c1a3b3ba 100644 --- a/tests/test_builders/test_build_html.py +++ b/tests/test_builders/test_build_html.py @@ -356,7 +356,8 @@ def test_html_signaturereturn_icon(app): assert ('→' in content) -@pytest.mark.sphinx('html', testroot='root', srcdir=os.urandom(4).hex()) +@pytest.mark.sphinx('html', testroot='root') +@pytest.mark.isolate() # because we change the sources in-place def test_html_remove_sources_before_write_gh_issue_10786(app, warning): # see: https://github.com/sphinx-doc/sphinx/issues/10786 target = app.srcdir / 'img.png' diff --git a/tests/test_builders/test_build_html_numfig.py b/tests/test_builders/test_build_html_numfig.py index 62e68cb6d82..4b2b6d8d892 100644 --- a/tests/test_builders/test_build_html_numfig.py +++ b/tests/test_builders/test_build_html_numfig.py @@ -61,6 +61,7 @@ def test_numfig_disabled(app, cached_etree_parse, fname, path, check, be_found): 'html', testroot='numfig', srcdir='test_numfig_without_numbered_toctree_warn', confoverrides={'numfig': True}) +@pytest.mark.isolate() # because we affect the sources def test_numfig_without_numbered_toctree_warn(app, warning): app.build() # remove :numbered: option @@ -145,6 +146,7 @@ def test_numfig_without_numbered_toctree_warn(app, warning): 'html', testroot='numfig', srcdir='test_numfig_without_numbered_toctree', confoverrides={'numfig': True}) +@pytest.mark.isolate('grouped') # because we affect the sources def test_numfig_without_numbered_toctree(app, cached_etree_parse, fname, path, check, be_found): # remove :numbered: option index = (app.srcdir / 'index.rst').read_text(encoding='utf8') diff --git a/tests/test_builders/test_build_latex.py b/tests/test_builders/test_build_latex.py index 2c3cae18bb5..a23dff6760f 100644 --- a/tests/test_builders/test_build_latex.py +++ b/tests/test_builders/test_build_latex.py @@ -1210,7 +1210,7 @@ def test_maxlistdepth_at_ten(app, status, warning): confoverrides={'latex_table_style': []}) @pytest.mark.test_params(shared_result='latex-table') def test_latex_table_tabulars(app, status, warning): - app.build(force_all=True) + app.build() result = (app.outdir / 'python.tex').read_text(encoding='utf8') tables = {} for chap in re.split(r'\\(?:section|chapter){', result)[1:]: @@ -1281,7 +1281,7 @@ def get_expected(name): confoverrides={'latex_table_style': []}) @pytest.mark.test_params(shared_result='latex-table') def test_latex_table_longtable(app, status, warning): - app.build(force_all=True) + app.build() result = (app.outdir / 'python.tex').read_text(encoding='utf8') tables = {} for chap in re.split(r'\\(?:section|chapter){', result)[1:]: @@ -1342,7 +1342,7 @@ def get_expected(name): confoverrides={'latex_table_style': []}) @pytest.mark.test_params(shared_result='latex-table') def test_latex_table_complex_tables(app, status, warning): - app.build(force_all=True) + app.build() result = (app.outdir / 'python.tex').read_text(encoding='utf8') tables = {} for chap in re.split(r'\\(?:section|renewcommand){', result)[1:]: @@ -1405,7 +1405,7 @@ def test_latex_table_custom_template_caseB(app, status, warning): @pytest.mark.sphinx('latex', testroot='latex-table') @pytest.mark.test_params(shared_result='latex-table') def test_latex_table_custom_template_caseC(app, status, warning): - app.build(force_all=True) + app.build() result = (app.outdir / 'python.tex').read_text(encoding='utf8') assert 'SALUT LES COPAINS' not in result diff --git a/tests/test_environment/test_environment.py b/tests/test_environment/test_environment.py index 8a34457cb90..e2c7930ab9b 100644 --- a/tests/test_environment/test_environment.py +++ b/tests/test_environment/test_environment.py @@ -15,7 +15,7 @@ def test_config_status(make_app, app_params): args, kwargs = app_params # clean build - app1 = make_app(*args, freshenv=True, **kwargs) + app1 = make_app(*args, **dict(kwargs, freshenv=True)) assert app1.env.config_status == CONFIG_NEW app1.build() assert '[new config] 1 added' in app1._status.getvalue() @@ -27,7 +27,7 @@ def test_config_status(make_app, app_params): assert "0 added, 0 changed, 0 removed" in app2._status.getvalue() # incremental build (config entry changed) - app3 = make_app(*args, confoverrides={'root_doc': 'indexx'}, **kwargs) + app3 = make_app(*args, **dict(kwargs, confoverrides={'root_doc': 'indexx'})) fname = os.path.join(app3.srcdir, 'index.rst') assert os.path.isfile(fname) shutil.move(fname, fname[:-4] + 'x.rst') @@ -37,7 +37,7 @@ def test_config_status(make_app, app_params): assert "[config changed ('root_doc')] 1 added" in app3._status.getvalue() # incremental build (extension changed) - app4 = make_app(*args, confoverrides={'extensions': ['sphinx.ext.autodoc']}, **kwargs) + app4 = make_app(*args, **dict(kwargs, confoverrides={'extensions': ['sphinx.ext.autodoc']})) assert app4.env.config_status == CONFIG_EXTENSIONS_CHANGED app4.build() want_str = "[extensions changed ('sphinx.ext.autodoc')] 1 added" diff --git a/tests/test_extensions/test_ext_autodoc.py b/tests/test_extensions/test_ext_autodoc.py index 5a2e91cb5e9..b6f7ed57c7b 100644 --- a/tests/test_extensions/test_ext_autodoc.py +++ b/tests/test_extensions/test_ext_autodoc.py @@ -10,6 +10,7 @@ import itertools import operator import sys +import uuid from types import SimpleNamespace from typing import TYPE_CHECKING from unittest.mock import Mock @@ -2492,7 +2493,9 @@ def test_singledispatchmethod_classmethod_automethod(app): @pytest.mark.skipif(sys.version_info[:2] >= (3, 13), reason='Cython does not support Python 3.13 yet.') @pytest.mark.skipif(pyximport is None, reason='cython is not installed') -@pytest.mark.sphinx('html', testroot='ext-autodoc') +# use an explicit 'srcdir' to make the path smaller on Windows platforms +# so that cython can correctly compile the files +@pytest.mark.sphinx('html', srcdir=uuid.uuid4().hex, testroot='ext-autodoc') def test_cython(app): options = {"members": None, "undoc-members": None} diff --git a/tests/test_extensions/test_ext_autodoc_configs.py b/tests/test_extensions/test_ext_autodoc_configs.py index 1262b15162b..820c1bcb823 100644 --- a/tests/test_extensions/test_ext_autodoc_configs.py +++ b/tests/test_extensions/test_ext_autodoc_configs.py @@ -1005,6 +1005,7 @@ def test_autodoc_typehints_description(app): @pytest.mark.sphinx('text', testroot='ext-autodoc', confoverrides={'autodoc_typehints': "description", 'autodoc_typehints_description_target': 'documented'}) +@pytest.mark.isolate() # because we change the sources in-place def test_autodoc_typehints_description_no_undoc(app): # No :type: or :rtype: will be injected for `incr`, which does not have # a description for its parameters or its return. `tuple_args` does @@ -1049,6 +1050,7 @@ def test_autodoc_typehints_description_no_undoc(app): @pytest.mark.sphinx('text', testroot='ext-autodoc', confoverrides={'autodoc_typehints': "description", 'autodoc_typehints_description_target': 'documented_params'}) +@pytest.mark.isolate() # because we change the sources in-place def test_autodoc_typehints_description_no_undoc_doc_rtype(app): # No :type: will be injected for `incr`, which does not have a description # for its parameters or its return, just :rtype: will be injected due to @@ -1113,6 +1115,7 @@ def test_autodoc_typehints_description_no_undoc_doc_rtype(app): @pytest.mark.sphinx('text', testroot='ext-autodoc', confoverrides={'autodoc_typehints': "description"}) +@pytest.mark.isolate() # because we change the sources in-place def test_autodoc_typehints_description_with_documented_init(app): with overwrite_file(app.srcdir / 'index.rst', '.. autoclass:: target.typehints._ClassWithDocumentedInit\n' @@ -1150,6 +1153,7 @@ def test_autodoc_typehints_description_with_documented_init(app): @pytest.mark.sphinx('text', testroot='ext-autodoc', confoverrides={'autodoc_typehints': "description", 'autodoc_typehints_description_target': 'documented'}) +@pytest.mark.isolate() # because we change the sources in-place def test_autodoc_typehints_description_with_documented_init_no_undoc(app): with overwrite_file(app.srcdir / 'index.rst', '.. autoclass:: target.typehints._ClassWithDocumentedInit\n' @@ -1177,6 +1181,7 @@ def test_autodoc_typehints_description_with_documented_init_no_undoc(app): @pytest.mark.sphinx('text', testroot='ext-autodoc', confoverrides={'autodoc_typehints': "description", 'autodoc_typehints_description_target': 'documented_params'}) +@pytest.mark.isolate() # because we change the sources in-place def test_autodoc_typehints_description_with_documented_init_no_undoc_doc_rtype(app): # see test_autodoc_typehints_description_with_documented_init_no_undoc # returnvalue_and_documented_params should not change class or method @@ -1213,6 +1218,7 @@ def test_autodoc_typehints_description_for_invalid_node(app): @pytest.mark.sphinx('text', testroot='ext-autodoc', confoverrides={'autodoc_typehints': "both"}) +@pytest.mark.isolate() # because we change the sources in-place def test_autodoc_typehints_both(app): with overwrite_file(app.srcdir / 'index.rst', '.. autofunction:: target.typehints.incr\n' @@ -1398,6 +1404,7 @@ def test_autodoc_type_aliases(app): srcdir='autodoc_typehints_description_and_type_aliases', confoverrides={'autodoc_typehints': "description", 'autodoc_type_aliases': {'myint': 'myint'}}) +@pytest.mark.isolate() # because we change the sources in-place def test_autodoc_typehints_description_and_type_aliases(app): with overwrite_file(app.srcdir / 'autodoc_type_aliases.rst', '.. autofunction:: target.autodoc_type_aliases.sum'): diff --git a/tests/test_intl/test_catalogs.py b/tests/test_intl/test_catalogs.py index b7fd7be6f1c..1b74bca86e6 100644 --- a/tests/test_intl/test_catalogs.py +++ b/tests/test_intl/test_catalogs.py @@ -26,10 +26,10 @@ def _setup_test(app_params): @pytest.mark.usefixtures('_setup_test') -@pytest.mark.test_params(shared_result='test-catalogs') @pytest.mark.sphinx( 'html', testroot='intl', confoverrides={'language': 'en', 'locale_dirs': ['./locale']}) +@pytest.mark.isolate() # for Windows def test_compile_all_catalogs(app, status, warning): app.builder.compile_all_catalogs() @@ -42,10 +42,10 @@ def test_compile_all_catalogs(app, status, warning): @pytest.mark.usefixtures('_setup_test') -@pytest.mark.test_params(shared_result='test-catalogs') @pytest.mark.sphinx( 'html', testroot='intl', confoverrides={'language': 'en', 'locale_dirs': ['./locale']}) +@pytest.mark.isolate() # for Windows def test_compile_specific_catalogs(app, status, warning): locale_dir = app.srcdir / 'locale' catalog_dir = locale_dir / app.config.language / 'LC_MESSAGES' @@ -59,10 +59,10 @@ def test_compile_specific_catalogs(app, status, warning): @pytest.mark.usefixtures('_setup_test') -@pytest.mark.test_params(shared_result='test-catalogs') @pytest.mark.sphinx( 'html', testroot='intl', confoverrides={'language': 'en', 'locale_dirs': ['./locale']}) +@pytest.mark.isolate() # for Windows def test_compile_update_catalogs(app, status, warning): app.builder.compile_update_catalogs() diff --git a/tests/test_testing/__init__.py b/tests/test_testing/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_testing/_const.py b/tests/test_testing/_const.py new file mode 100644 index 00000000000..f209b8bbdd5 --- /dev/null +++ b/tests/test_testing/_const.py @@ -0,0 +1,16 @@ +"""Private constants.""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +import sphinx + +if TYPE_CHECKING: + from typing import Final + +PROJECT_PATH: Final[str] = os.path.realpath(os.path.dirname(os.path.dirname(sphinx.__file__))) +"""Directory containing the current (local) sphinx's implementation.""" + +SPHINX_PLUGIN_NAME: Final[str] = 'sphinx.testing.fixtures' diff --git a/tests/test_testing/conftest.py b/tests/test_testing/conftest.py new file mode 100644 index 00000000000..a2fea0b1ea2 --- /dev/null +++ b/tests/test_testing/conftest.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +import pytest + +from ._const import PROJECT_PATH, SPHINX_PLUGIN_NAME + +if TYPE_CHECKING: + from _pytest.config import Config + from _pytest.pytester import Pytester + +pytest_plugins = ['pytester'] + + +# change this fixture when the rest of the test suite is changed +@pytest.fixture(scope='package') +def default_testroot(): + return 'minimal' + + +@pytest.fixture(autouse=True) +def _pytester_pyprojecttoml(pytester: Pytester) -> None: + # TL;DR: this is a patch to force pytester & xdist using the local plugin + # implementation and not a possibly out-of-date installed version. + # + # :mod:`xdist.plugin` contains a snapshot of ``sys.path`` which is then + # passed to the workers; however, the snapshot is created when the module + # is imported. Apparently, even if ``pytester.syspathinsert(...)`` is used, + # the ``xdist.plugin`` module is not reloaded and thus the snapshot is not + # correctly updated, meaning that when ``pytester`` effectively runs a test + # the ``xdist`` workers are not able to find the local implementation. + # + # In addition :mod:`xdist.remote` ignores a user-defined PYTHONPATH when + # setting its own ``sys.path``, leading to ``ImportError`` at runtime. + # + # Note that PyCharm (and probably other IDEs or tools) does not suffer from + # this since it can be configured to automatically extend ``sys.path`` with + # the project's sources. The issue seems to only appear when ``pytest`` is + # directly invoked from the CLI. + pytester.makepyprojecttoml(f""" +[tool.pytest.ini_options] +addopts = ["--import-mode=prepend", "--strict-config", "--strict-markers"] +pythonpath = [{PROJECT_PATH!r}] +xfail_strict = true +""") + + +@pytest.fixture(autouse=True) +def _pytester_conftest(pytestconfig: Config, pytester: Pytester) -> None: + testroot_dir = os.path.join(pytestconfig.rootpath, 'tests', 'roots') + pytester.makeconftest(f""" +import pytest + +pytest_plugins = [{SPHINX_PLUGIN_NAME!r}] +collect_ignore = ['certs', 'roots'] + +@pytest.fixture() +def sphinx_use_legacy_plugin() -> bool: # xref RemovedInSphinx90Warning + return False # use the new implementation + +@pytest.fixture(scope='session') +def rootdir(): + return {testroot_dir!r} + +@pytest.fixture(scope='session') +def default_testroot(): + return 'minimal' +""") diff --git a/tests/test_testing/test_plugin_markers.py b/tests/test_testing/test_plugin_markers.py new file mode 100644 index 00000000000..20a9a85438e --- /dev/null +++ b/tests/test_testing/test_plugin_markers.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from enum import IntEnum +from pathlib import Path + +import pytest + + +@pytest.mark.sphinx() +def test_mark_sphinx_use_default_builder(app_params): + args, kwargs = app_params + assert not args + assert kwargs['buildername'] == 'html' + + +@pytest.mark.sphinx('dummy') +def test_mark_sphinx_with_builder(app_params): + args, kwargs = app_params + assert not args + + testroot_path = kwargs['testroot_path'] + assert testroot_path is None or isinstance(testroot_path, str) + assert kwargs['shared_result'] is None + + assert kwargs['buildername'] == 'dummy' + assert kwargs['testroot'] == 'minimal' + assert isinstance(kwargs['srcdir'], Path) + assert kwargs['srcdir'].name == 'minimal' + + +@pytest.mark.parametrize( + ('sphinx_isolation', 'policy'), + [ + (False, 'minimal'), + (True, 'always'), + ('minimal', 'minimal'), + ('grouped', 'grouped'), + ('always', 'always'), + ], +) +@pytest.mark.sphinx('dummy') +def test_mark_sphinx_with_isolation(app_params, sphinx_isolation, policy): + isolate = app_params.kwargs['isolate'] + assert isinstance(isolate, IntEnum) + assert isolate.name == policy + + +@pytest.mark.sphinx('dummy') +@pytest.mark.test_params() +def test_mark_sphinx_with_implicit_shared_result(app_params, test_params): + shared_result = app_params.kwargs['shared_result'] + assert shared_result == test_params['shared_result'] + + srcdir = app_params.kwargs['srcdir'] + assert srcdir.name == f'minimal-{shared_result}' + + +@pytest.mark.sphinx('dummy') +@pytest.mark.test_params(shared_result='abc123') +def test_mark_sphinx_with_explicit_shared_result(app_params, test_params): + shared_result = app_params.kwargs['shared_result'] + assert shared_result == test_params['shared_result'] + assert shared_result == 'abc123' + + srcdir = app_params.kwargs['srcdir'] + assert srcdir.name == 'minimal-abc123' diff --git a/tests/test_testing/test_testroot_finder.py b/tests/test_testing/test_testroot_finder.py new file mode 100644 index 00000000000..ccb6bdfc534 --- /dev/null +++ b/tests/test_testing/test_testroot_finder.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import contextlib +import os +from typing import TYPE_CHECKING, overload + +import pytest +from _pytest.scope import Scope + +from sphinx.testing._internal.pytest_util import TestRootFinder + +if TYPE_CHECKING: + from typing import Any, Literal + + +def test_testroot_finder_empty_args(): + with pytest.raises(ValueError, match="expecting a non-empty string or None for 'path'"): + TestRootFinder('') + + with pytest.raises(ValueError, match="expecting a non-empty string or None for 'default'"): + TestRootFinder(None, None, '') + + with pytest.raises(ValueError, match="expecting a non-empty string or None for 'default'"): + # prefix is allowed to be '' + TestRootFinder(None, '', '') + + +@pytest.mark.parametrize('prefix', [None, '', 'test-']) +@pytest.mark.parametrize('default', [None, 'default']) +@pytest.mark.parametrize('name', [None, 'foo']) +def test_testroot_finder_no_rootdir(prefix, default, name): + finder = TestRootFinder(prefix=prefix, default=default) + assert finder.find(name) is None + + +@pytest.mark.parametrize('path', ['/']) +@pytest.mark.parametrize('prefix', [None, '']) +@pytest.mark.parametrize(('name', 'expect'), [('foo', '/foo'), (None, None)]) +def test_testroot_finder_no_default_no_prefix(path, prefix, name, expect): + finder = TestRootFinder(path, prefix) + assert finder.find(name) == expect + + +@pytest.mark.parametrize('path', ['/']) +@pytest.mark.parametrize('prefix', [None, '']) +@pytest.mark.parametrize('default', ['default']) +@pytest.mark.parametrize(('name', 'expect'), [('foo', '/foo'), (None, '/default')]) +def test_testroot_finder_with_default(path, prefix, default, name, expect): + finder = TestRootFinder(path, prefix, default) + assert finder.find(name) == expect + + +@pytest.mark.parametrize('path', ['/']) +@pytest.mark.parametrize('prefix', ['test-']) +@pytest.mark.parametrize(('name', 'expect'), [('foo', '/test-foo'), (None, None)]) +def test_testroot_finder_with_prefix(path, prefix, name, expect): + finder = TestRootFinder(path, prefix) + assert finder.find(name) == expect + + +@pytest.mark.parametrize('path', ['/']) +@pytest.mark.parametrize('prefix', ['test-']) +@pytest.mark.parametrize('default', ['default']) +@pytest.mark.parametrize(('name', 'expect'), [('foo', '/test-foo'), (None, '/test-default')]) +def test_testroot_finder(pytester, path, prefix, default, name, expect): + finder = TestRootFinder(path, prefix, default) + assert finder.find(name) == expect + + +############################################################################### +# E2E tests +############################################################################### + + +# fmt: off +@overload +def e2e_with_fixture_def( # NoQA: E704 + fixt: Literal['rootdir'], attr: Literal['path'], + value: str | os.PathLike[str] | None, expect: str | None, + scope: Scope, +) -> str: ... +@overload # NoQA: E302 +def e2e_with_fixture_def( # NoQA: E704 + fixt: Literal['testroot_prefix'], attr: Literal['prefix'], + value: str | None, expect: str, + scope: Scope, +) -> str: ... +@overload # NoQA: E302 +def e2e_with_fixture_def( # NoQA: E704 + fixt: Literal['default_testroot'], attr: Literal['default'], + value: str | None, expect: str | None, + scope: Scope, +) -> str: ... +# fmt: on +def e2e_with_fixture_def( # NoQA: E302 + fixt: str, attr: str, value: Any, expect: Any, scope: Scope +) -> str: + """A test with an attribute defined via a fixture. + + :param fixt: The fixture name. + :param attr: An attribute name. + :param value: The return value of the fixture. + :param expect: The expected attribute value. + :param scope: The fixture scope. + :return: The test file source. + """ + return f""" +import pytest + +@pytest.fixture(scope={scope.value!r}) +def {fixt}(): + return {value!r} + +def test(testroot_finder, {fixt}): + assert {fixt} == {value!r} + assert testroot_finder.{attr} == {expect!r} +""" + + +# fmt: off +@overload +def e2e_with_parametrize( # NoQA: E704 + fixt: Literal['rootdir'], attr: Literal['path'], + value: str | os.PathLike[str] | None, expect: str | None, + scope: Scope, +) -> str: ... +@overload # NoQA: E302 +def e2e_with_parametrize( # NoQA: E704 + fixt: Literal['testroot_prefix'], attr: Literal['prefix'], + value: str | None, expect: str, + scope: Scope, +) -> str: ... +@overload # NoQA: E302 +def e2e_with_parametrize( # NoQA: E704 + fixt: Literal['default_testroot'], attr: Literal['default'], + value: str | None, expect: str | None, + scope: Scope, +) -> str: ... +# fmt: on +def e2e_with_parametrize( # NoQA: E302 + fixt: str, attr: str, value: Any, expect: Any, scope: Scope +) -> str: + """A test with an attribute defined via parametrization.""" + return f""" +import pytest + +@pytest.mark.parametrize({fixt!r}, [{value!r}], scope={scope.value!r}) +def test(testroot_finder, {fixt}): + assert {fixt} == {value!r} + assert testroot_finder.{attr} == {expect!r} +""" + + +@pytest.mark.parametrize('scope', Scope) +@pytest.mark.parametrize('value', [None, '/']) +def test_rootdir_e2e(pytester, scope, value): + script1 = e2e_with_fixture_def('rootdir', 'path', value, value, scope) + script2 = e2e_with_parametrize('rootdir', 'path', value, value, scope) + pytester.makepyfile(test_fixture_def=script1, test_parametrize=script2) + with open(os.devnull, 'w', encoding='utf-8') as NUL, contextlib.redirect_stdout(NUL): + res = pytester.runpytest_inprocess('-p no:xdist') + res.assert_outcomes(passed=2) + + +@pytest.mark.parametrize('scope', Scope) +@pytest.mark.parametrize('value', ['my-', '', None]) +def test_testroot_prefix_e2e(pytester, scope, value): + expect = value or '' # the constructor of TestRootFinder normalizes the prefix + script1 = e2e_with_fixture_def('testroot_prefix', 'prefix', value, expect, scope) + script2 = e2e_with_parametrize('testroot_prefix', 'prefix', value, expect, scope) + pytester.makepyfile(test_fixture_def=script1, test_parametrize=script2) + with open(os.devnull, 'w', encoding='utf-8') as NUL, contextlib.redirect_stdout(NUL): + res = pytester.runpytest_inprocess('-p no:xdist') + res.assert_outcomes(passed=2) + + +@pytest.mark.parametrize('scope', Scope) +@pytest.mark.parametrize('value', [None, 'default']) +def test_default_testroot_e2e(pytester, scope, value): + script1 = e2e_with_fixture_def('default_testroot', 'default', value, value, scope) + script2 = e2e_with_parametrize('default_testroot', 'default', value, value, scope) + pytester.makepyfile(test_fixture_def=script1, test_parametrize=script2) + with open(os.devnull, 'w', encoding='utf-8') as NUL, contextlib.redirect_stdout(NUL): + res = pytester.runpytest_inprocess('-p no:xdist') + res.assert_outcomes(passed=2) diff --git a/tests/test_toctree.py b/tests/test_toctree.py index e59085d4aca..39f7d25d37c 100644 --- a/tests/test_toctree.py +++ b/tests/test_toctree.py @@ -31,6 +31,7 @@ def test_singlehtml_toctree(app, status, warning): @pytest.mark.sphinx(testroot='toctree', srcdir="numbered-toctree") +@pytest.mark.isolate() # because we change the sources in-place def test_numbered_toctree(app, status, warning): # give argument to :numbered: option index = (app.srcdir / 'index.rst').read_text(encoding='utf8')