From b706a2c04840a8057610f41071fbcf3da1290eb5 Mon Sep 17 00:00:00 2001 From: Tadeu Manoel Date: Mon, 5 Apr 2021 17:10:03 -0300 Subject: [PATCH] Fix error with --import-mode=importlib and modules containing dataclasses or pickle (#7870) Co-authored-by: Bruno Oliveira Fixes #7856, fixes #7859 --- AUTHORS | 1 + changelog/7856.feature.rst | 2 + doc/en/explanation/goodpractices.rst | 2 +- doc/en/explanation/pythonpath.rst | 19 ++- src/_pytest/config/__init__.py | 39 +++--- src/_pytest/doctest.py | 6 +- src/_pytest/main.py | 10 +- src/_pytest/pathlib.py | 52 ++++++- src/_pytest/python.py | 2 +- testing/code/test_excinfo.py | 4 +- testing/code/test_source.py | 2 +- testing/python/fixtures.py | 6 +- testing/test_config.py | 10 +- testing/test_conftest.py | 78 ++++++++--- testing/test_pathlib.py | 201 ++++++++++++++++++++++----- testing/test_pluginmanager.py | 20 ++- 16 files changed, 348 insertions(+), 106 deletions(-) create mode 100644 changelog/7856.feature.rst diff --git a/AUTHORS b/AUTHORS index e5477e1f1a5..46f283452c3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -98,6 +98,7 @@ Dominic Mortlock Duncan Betts Edison Gustavo Muenz Edoardo Batini +Edson Tadeu M. Manoel Eduardo Schettino Eli Boyarski Elizaveta Shashkova diff --git a/changelog/7856.feature.rst b/changelog/7856.feature.rst new file mode 100644 index 00000000000..22ed4c83bc3 --- /dev/null +++ b/changelog/7856.feature.rst @@ -0,0 +1,2 @@ +:ref:`--import-mode=importlib ` now works with features that +depend on modules being on :py:data:`sys.modules`, such as :mod:`pickle` and :mod:`dataclasses`. diff --git a/doc/en/explanation/goodpractices.rst b/doc/en/explanation/goodpractices.rst index 77c176182fc..fe5db9fd1eb 100644 --- a/doc/en/explanation/goodpractices.rst +++ b/doc/en/explanation/goodpractices.rst @@ -151,7 +151,7 @@ This layout prevents a lot of common pitfalls and has many benefits, which are b .. note:: The new ``--import-mode=importlib`` (see :ref:`import-modes`) doesn't have - any of the drawbacks above because ``sys.path`` and ``sys.modules`` are not changed when importing + any of the drawbacks above because ``sys.path`` is not changed when importing test modules, so users that run into this issue are strongly encouraged to try it and report if the new option works well for them. diff --git a/doc/en/explanation/pythonpath.rst b/doc/en/explanation/pythonpath.rst index 6aac3bc86f2..9aef44618fc 100644 --- a/doc/en/explanation/pythonpath.rst +++ b/doc/en/explanation/pythonpath.rst @@ -16,14 +16,14 @@ import process can be controlled through the ``--import-mode`` command-line flag these values: * ``prepend`` (default): the directory path containing each module will be inserted into the *beginning* - of ``sys.path`` if not already there, and then imported with the `__import__ `__ builtin. + of :py:data:`sys.path` if not already there, and then imported with the `__import__ `__ builtin. This requires test module names to be unique when the test directory tree is not arranged in - packages, because the modules will put in ``sys.modules`` after importing. + packages, because the modules will put in :py:data:`sys.modules` after importing. This is the classic mechanism, dating back from the time Python 2 was still supported. -* ``append``: the directory containing each module is appended to the end of ``sys.path`` if not already +* ``append``: the directory containing each module is appended to the end of :py:data:`sys.path` if not already there, and imported with ``__import__``. This better allows to run test modules against installed versions of a package even if the @@ -41,17 +41,14 @@ these values: we advocate for using :ref:`src ` layouts. Same as ``prepend``, requires test module names to be unique when the test directory tree is - not arranged in packages, because the modules will put in ``sys.modules`` after importing. + not arranged in packages, because the modules will put in :py:data:`sys.modules` after importing. -* ``importlib``: new in pytest-6.0, this mode uses `importlib `__ to import test modules. This gives full control over the import process, and doesn't require - changing ``sys.path`` or ``sys.modules`` at all. +* ``importlib``: new in pytest-6.0, this mode uses `importlib `__ to import test modules. This gives full control over the import process, and doesn't require changing :py:data:`sys.path`. - For this reason this doesn't require test module names to be unique at all, but also makes test - modules non-importable by each other. This was made possible in previous modes, for tests not residing - in Python packages, because of the side-effects of changing ``sys.path`` and ``sys.modules`` - mentioned above. Users which require this should turn their tests into proper packages instead. + For this reason this doesn't require test module names to be unique, but also makes test + modules non-importable by each other. - We intend to make ``importlib`` the default in future releases. + We intend to make ``importlib`` the default in future releases, depending on feedback. ``prepend`` and ``append`` import modes scenarios ------------------------------------------------- diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 12e73859d29..23fa3e3d5db 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -481,7 +481,9 @@ def pytest_configure(self, config: "Config") -> None: # # Internal API for local conftest plugin handling. # - def _set_initial_conftests(self, namespace: argparse.Namespace) -> None: + def _set_initial_conftests( + self, namespace: argparse.Namespace, rootpath: Path + ) -> None: """Load initial conftest files given a preparsed "namespace". As conftest files may add their own command line options which have @@ -507,26 +509,24 @@ def _set_initial_conftests(self, namespace: argparse.Namespace) -> None: path = path[:i] anchor = absolutepath(current / path) if anchor.exists(): # we found some file object - self._try_load_conftest(anchor, namespace.importmode) + self._try_load_conftest(anchor, namespace.importmode, rootpath) foundanchor = True if not foundanchor: - self._try_load_conftest(current, namespace.importmode) + self._try_load_conftest(current, namespace.importmode, rootpath) def _try_load_conftest( - self, anchor: Path, importmode: Union[str, ImportMode] + self, anchor: Path, importmode: Union[str, ImportMode], rootpath: Path ) -> None: - self._getconftestmodules(anchor, importmode) + self._getconftestmodules(anchor, importmode, rootpath) # let's also consider test* subdirs if anchor.is_dir(): for x in anchor.glob("test*"): if x.is_dir(): - self._getconftestmodules(x, importmode) + self._getconftestmodules(x, importmode, rootpath) @lru_cache(maxsize=128) def _getconftestmodules( - self, - path: Path, - importmode: Union[str, ImportMode], + self, path: Path, importmode: Union[str, ImportMode], rootpath: Path ) -> List[types.ModuleType]: if self._noconftest: return [] @@ -545,7 +545,7 @@ def _getconftestmodules( continue conftestpath = parent / "conftest.py" if conftestpath.is_file(): - mod = self._importconftest(conftestpath, importmode) + mod = self._importconftest(conftestpath, importmode, rootpath) clist.append(mod) self._dirpath2confmods[directory] = clist return clist @@ -555,8 +555,9 @@ def _rget_with_confmod( name: str, path: Path, importmode: Union[str, ImportMode], + rootpath: Path, ) -> Tuple[types.ModuleType, Any]: - modules = self._getconftestmodules(path, importmode) + modules = self._getconftestmodules(path, importmode, rootpath=rootpath) for mod in reversed(modules): try: return mod, getattr(mod, name) @@ -565,9 +566,7 @@ def _rget_with_confmod( raise KeyError(name) def _importconftest( - self, - conftestpath: Path, - importmode: Union[str, ImportMode], + self, conftestpath: Path, importmode: Union[str, ImportMode], rootpath: Path ) -> types.ModuleType: # Use a resolved Path object as key to avoid loading the same conftest # twice with build systems that create build directories containing @@ -584,7 +583,7 @@ def _importconftest( _ensure_removed_sysmodule(conftestpath.stem) try: - mod = import_path(conftestpath, mode=importmode) + mod = import_path(conftestpath, mode=importmode, root=rootpath) except Exception as e: assert e.__traceback__ is not None exc_info = (type(e), e, e.__traceback__) @@ -1086,7 +1085,9 @@ def _processopt(self, opt: "Argument") -> None: @hookimpl(trylast=True) def pytest_load_initial_conftests(self, early_config: "Config") -> None: - self.pluginmanager._set_initial_conftests(early_config.known_args_namespace) + self.pluginmanager._set_initial_conftests( + early_config.known_args_namespace, rootpath=early_config.rootpath + ) def _initini(self, args: Sequence[str]) -> None: ns, unknown_args = self._parser.parse_known_and_unknown_args( @@ -1437,10 +1438,12 @@ def _getini(self, name: str): assert type in [None, "string"] return value - def _getconftest_pathlist(self, name: str, path: Path) -> Optional[List[Path]]: + def _getconftest_pathlist( + self, name: str, path: Path, rootpath: Path + ) -> Optional[List[Path]]: try: mod, relroots = self.pluginmanager._rget_with_confmod( - name, path, self.getoption("importmode") + name, path, self.getoption("importmode"), rootpath ) except KeyError: return None diff --git a/src/_pytest/doctest.py b/src/_pytest/doctest.py index 5eaeccc4766..9b877cfd9b2 100644 --- a/src/_pytest/doctest.py +++ b/src/_pytest/doctest.py @@ -534,11 +534,13 @@ def _find( if self.path.name == "conftest.py": module = self.config.pluginmanager._importconftest( - self.path, self.config.getoption("importmode") + self.path, + self.config.getoption("importmode"), + rootpath=self.config.rootpath, ) else: try: - module = import_path(self.path) + module = import_path(self.path, root=self.config.rootpath) except ImportError: if self.config.getvalue("doctest_ignore_import_errors"): pytest.skip("unable to import module %r" % self.path) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 06cfb1fd54f..dda6c557e02 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -378,7 +378,9 @@ def _in_venv(path: Path) -> bool: def pytest_ignore_collect(fspath: Path, config: Config) -> Optional[bool]: - ignore_paths = config._getconftest_pathlist("collect_ignore", path=fspath.parent) + ignore_paths = config._getconftest_pathlist( + "collect_ignore", path=fspath.parent, rootpath=config.rootpath + ) ignore_paths = ignore_paths or [] excludeopt = config.getoption("ignore") if excludeopt: @@ -388,7 +390,7 @@ def pytest_ignore_collect(fspath: Path, config: Config) -> Optional[bool]: return True ignore_globs = config._getconftest_pathlist( - "collect_ignore_glob", path=fspath.parent + "collect_ignore_glob", path=fspath.parent, rootpath=config.rootpath ) ignore_globs = ignore_globs or [] excludeglobopt = config.getoption("ignore_glob") @@ -546,7 +548,9 @@ def gethookproxy(self, fspath: "os.PathLike[str]"): # hooks with all conftest.py files. pm = self.config.pluginmanager my_conftestmodules = pm._getconftestmodules( - Path(fspath), self.config.getoption("importmode") + Path(fspath), + self.config.getoption("importmode"), + rootpath=self.config.rootpath, ) remove_mods = pm._conftest_plugins.difference(my_conftestmodules) if remove_mods: diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index e86811f1250..a0aa800f7ee 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -23,6 +23,7 @@ from posixpath import sep as posix_sep from types import ModuleType from typing import Callable +from typing import Dict from typing import Iterable from typing import Iterator from typing import Optional @@ -454,6 +455,7 @@ def import_path( p: Union[str, "os.PathLike[str]"], *, mode: Union[str, ImportMode] = ImportMode.prepend, + root: Path, ) -> ModuleType: """Import and return a module from the given path, which can be a file (a module) or a directory (a package). @@ -471,6 +473,11 @@ def import_path( to import the module, which avoids having to use `__import__` and muck with `sys.path` at all. It effectively allows having same-named test modules in different places. + :param root: + Used as an anchor when mode == ImportMode.importlib to obtain + a unique name for the module being imported so it can safely be stored + into ``sys.modules``. + :raises ImportPathMismatchError: If after importing the given `path` and the module `__file__` are different. Only raised in `prepend` and `append` modes. @@ -483,7 +490,7 @@ def import_path( raise ImportError(path) if mode is ImportMode.importlib: - module_name = path.stem + module_name = module_name_from_path(path, root) for meta_importer in sys.meta_path: spec = meta_importer.find_spec(module_name, [str(path.parent)]) @@ -497,7 +504,9 @@ def import_path( "Can't find module {} at location {}".format(module_name, str(path)) ) mod = importlib.util.module_from_spec(spec) + sys.modules[module_name] = mod spec.loader.exec_module(mod) # type: ignore[union-attr] + insert_missing_modules(sys.modules, module_name) return mod pkg_path = resolve_package_path(path) @@ -562,6 +571,47 @@ def _is_same(f1: str, f2: str) -> bool: return os.path.samefile(f1, f2) +def module_name_from_path(path: Path, root: Path) -> str: + """ + Return a dotted module name based on the given path, anchored on root. + + For example: path="projects/src/tests/test_foo.py" and root="/projects", the + resulting module name will be "src.tests.test_foo". + """ + path = path.with_suffix("") + try: + relative_path = path.relative_to(root) + except ValueError: + # If we can't get a relative path to root, use the full path, except + # for the first part ("d:\\" or "/" depending on the platform, for example). + path_parts = path.parts[1:] + else: + # Use the parts for the relative path to the root path. + path_parts = relative_path.parts + + return ".".join(path_parts) + + +def insert_missing_modules(modules: Dict[str, ModuleType], module_name: str) -> None: + """ + Used by ``import_path`` to create intermediate modules when using mode=importlib. + + When we want to import a module as "src.tests.test_foo" for example, we need + to create empty modules "src" and "src.tests" after inserting "src.tests.test_foo", + otherwise "src.tests.test_foo" is not importable by ``__import__``. + """ + module_parts = module_name.split(".") + while module_name: + if module_name not in modules: + module = ModuleType( + module_name, + doc="Empty module created by pytest's importmode=importlib.", + ) + modules[module_name] = module + module_parts.pop(-1) + module_name = ".".join(module_parts) + + def resolve_package_path(path: Path) -> Optional[Path]: """Return the Python package path by looking for the last directory upwards which still contains an __init__.py. diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 04fbb45701b..eec93fc0396 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -577,7 +577,7 @@ def _importtestmodule(self): # We assume we are only called once per module. importmode = self.config.getoption("--import-mode") try: - mod = import_path(self.path, mode=importmode) + mod = import_path(self.path, mode=importmode, root=self.config.rootpath) except SyntaxError as e: raise self.CollectError( ExceptionInfo.from_current().getrepr(style="short") diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py index e6a9cbaf737..11cf3e60e2d 100644 --- a/testing/code/test_excinfo.py +++ b/testing/code/test_excinfo.py @@ -162,7 +162,7 @@ def test_traceback_cut(self) -> None: def test_traceback_cut_excludepath(self, pytester: Pytester) -> None: p = pytester.makepyfile("def f(): raise ValueError") with pytest.raises(ValueError) as excinfo: - import_path(p).f() # type: ignore[attr-defined] + import_path(p, root=pytester.path).f() # type: ignore[attr-defined] basedir = Path(pytest.__file__).parent newtraceback = excinfo.traceback.cut(excludepath=basedir) for x in newtraceback: @@ -443,7 +443,7 @@ def importasmod(source): tmp_path.joinpath("__init__.py").touch() modpath.write_text(source) importlib.invalidate_caches() - return import_path(modpath) + return import_path(modpath, root=tmp_path) return importasmod diff --git a/testing/code/test_source.py b/testing/code/test_source.py index 5f2c6b1ea54..53e1bb9856b 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -298,7 +298,7 @@ def method(self): ) path = tmp_path.joinpath("a.py") path.write_text(str(source)) - mod: Any = import_path(path) + mod: Any = import_path(path, root=tmp_path) s2 = Source(mod.A) assert str(source).strip() == str(s2).strip() diff --git a/testing/python/fixtures.py b/testing/python/fixtures.py index 3eb4b80f39e..569b5d67e26 100644 --- a/testing/python/fixtures.py +++ b/testing/python/fixtures.py @@ -2069,9 +2069,9 @@ def test_2(self): reprec = pytester.inline_run("-v", "-s", "--confcutdir", pytester.path) reprec.assertoutcome(passed=8) config = reprec.getcalls("pytest_unconfigure")[0].config - values = config.pluginmanager._getconftestmodules(p, importmode="prepend")[ - 0 - ].values + values = config.pluginmanager._getconftestmodules( + p, importmode="prepend", rootpath=pytester.path + )[0].values assert values == ["fin_a1", "fin_a2", "fin_b1", "fin_b2"] * 2 def test_scope_ordering(self, pytester: Pytester) -> None: diff --git a/testing/test_config.py b/testing/test_config.py index 61fea3643b9..1d1a80aa4ad 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -597,8 +597,14 @@ def test_getconftest_pathlist(self, pytester: Pytester, tmp_path: Path) -> None: p = tmp_path.joinpath("conftest.py") p.write_text(f"pathlist = ['.', {str(somepath)!r}]") config = pytester.parseconfigure(p) - assert config._getconftest_pathlist("notexist", path=tmp_path) is None - pl = config._getconftest_pathlist("pathlist", path=tmp_path) or [] + assert ( + config._getconftest_pathlist("notexist", path=tmp_path, rootpath=tmp_path) + is None + ) + pl = ( + config._getconftest_pathlist("pathlist", path=tmp_path, rootpath=tmp_path) + or [] + ) print(pl) assert len(pl) == 2 assert pl[0] == tmp_path diff --git a/testing/test_conftest.py b/testing/test_conftest.py index 344c9bc5162..3ccdeb96495 100644 --- a/testing/test_conftest.py +++ b/testing/test_conftest.py @@ -35,7 +35,7 @@ def __init__(self) -> None: self.importmode = "prepend" namespace = cast(argparse.Namespace, Namespace()) - conftest._set_initial_conftests(namespace) + conftest._set_initial_conftests(namespace, rootpath=Path(args[0])) @pytest.mark.usefixtures("_sys_snapshot") @@ -57,39 +57,62 @@ def basedir( def test_basic_init(self, basedir: Path) -> None: conftest = PytestPluginManager() p = basedir / "adir" - assert conftest._rget_with_confmod("a", p, importmode="prepend")[1] == 1 + assert ( + conftest._rget_with_confmod("a", p, importmode="prepend", rootpath=basedir)[ + 1 + ] + == 1 + ) def test_immediate_initialiation_and_incremental_are_the_same( self, basedir: Path ) -> None: conftest = PytestPluginManager() assert not len(conftest._dirpath2confmods) - conftest._getconftestmodules(basedir, importmode="prepend") + conftest._getconftestmodules( + basedir, importmode="prepend", rootpath=Path(basedir) + ) snap1 = len(conftest._dirpath2confmods) assert snap1 == 1 - conftest._getconftestmodules(basedir / "adir", importmode="prepend") + conftest._getconftestmodules( + basedir / "adir", importmode="prepend", rootpath=basedir + ) assert len(conftest._dirpath2confmods) == snap1 + 1 - conftest._getconftestmodules(basedir / "b", importmode="prepend") + conftest._getconftestmodules( + basedir / "b", importmode="prepend", rootpath=basedir + ) assert len(conftest._dirpath2confmods) == snap1 + 2 def test_value_access_not_existing(self, basedir: Path) -> None: conftest = ConftestWithSetinitial(basedir) with pytest.raises(KeyError): - conftest._rget_with_confmod("a", basedir, importmode="prepend") + conftest._rget_with_confmod( + "a", basedir, importmode="prepend", rootpath=Path(basedir) + ) def test_value_access_by_path(self, basedir: Path) -> None: conftest = ConftestWithSetinitial(basedir) adir = basedir / "adir" - assert conftest._rget_with_confmod("a", adir, importmode="prepend")[1] == 1 assert ( - conftest._rget_with_confmod("a", adir / "b", importmode="prepend")[1] == 1.5 + conftest._rget_with_confmod( + "a", adir, importmode="prepend", rootpath=basedir + )[1] + == 1 + ) + assert ( + conftest._rget_with_confmod( + "a", adir / "b", importmode="prepend", rootpath=basedir + )[1] + == 1.5 ) def test_value_access_with_confmod(self, basedir: Path) -> None: startdir = basedir / "adir" / "b" startdir.joinpath("xx").mkdir() conftest = ConftestWithSetinitial(startdir) - mod, value = conftest._rget_with_confmod("a", startdir, importmode="prepend") + mod, value = conftest._rget_with_confmod( + "a", startdir, importmode="prepend", rootpath=Path(basedir) + ) assert value == 1.5 path = Path(mod.__file__) assert path.parent == basedir / "adir" / "b" @@ -110,7 +133,9 @@ def test_doubledash_considered(pytester: Pytester) -> None: conf.joinpath("conftest.py").touch() conftest = PytestPluginManager() conftest_setinitial(conftest, [conf.name, conf.name]) - values = conftest._getconftestmodules(conf, importmode="prepend") + values = conftest._getconftestmodules( + conf, importmode="prepend", rootpath=pytester.path + ) assert len(values) == 1 @@ -134,7 +159,7 @@ def test_conftest_global_import(pytester: Pytester) -> None: import pytest from _pytest.config import PytestPluginManager conf = PytestPluginManager() - mod = conf._importconftest(Path("conftest.py"), importmode="prepend") + mod = conf._importconftest(Path("conftest.py"), importmode="prepend", rootpath=Path.cwd()) assert mod.x == 3 import conftest assert conftest is mod, (conftest, mod) @@ -142,7 +167,7 @@ def test_conftest_global_import(pytester: Pytester) -> None: sub.mkdir() subconf = sub / "conftest.py" subconf.write_text("y=4") - mod2 = conf._importconftest(subconf, importmode="prepend") + mod2 = conf._importconftest(subconf, importmode="prepend", rootpath=Path.cwd()) assert mod != mod2 assert mod2.y == 4 import conftest @@ -158,17 +183,25 @@ def test_conftestcutdir(pytester: Pytester) -> None: p = pytester.mkdir("x") conftest = PytestPluginManager() conftest_setinitial(conftest, [pytester.path], confcutdir=p) - values = conftest._getconftestmodules(p, importmode="prepend") + values = conftest._getconftestmodules( + p, importmode="prepend", rootpath=pytester.path + ) assert len(values) == 0 - values = conftest._getconftestmodules(conf.parent, importmode="prepend") + values = conftest._getconftestmodules( + conf.parent, importmode="prepend", rootpath=pytester.path + ) assert len(values) == 0 assert Path(conf) not in conftest._conftestpath2mod # but we can still import a conftest directly - conftest._importconftest(conf, importmode="prepend") - values = conftest._getconftestmodules(conf.parent, importmode="prepend") + conftest._importconftest(conf, importmode="prepend", rootpath=pytester.path) + values = conftest._getconftestmodules( + conf.parent, importmode="prepend", rootpath=pytester.path + ) assert values[0].__file__.startswith(str(conf)) # and all sub paths get updated properly - values = conftest._getconftestmodules(p, importmode="prepend") + values = conftest._getconftestmodules( + p, importmode="prepend", rootpath=pytester.path + ) assert len(values) == 1 assert values[0].__file__.startswith(str(conf)) @@ -177,7 +210,9 @@ def test_conftestcutdir_inplace_considered(pytester: Pytester) -> None: conf = pytester.makeconftest("") conftest = PytestPluginManager() conftest_setinitial(conftest, [conf.parent], confcutdir=conf.parent) - values = conftest._getconftestmodules(conf.parent, importmode="prepend") + values = conftest._getconftestmodules( + conf.parent, importmode="prepend", rootpath=pytester.path + ) assert len(values) == 1 assert values[0].__file__.startswith(str(conf)) @@ -347,13 +382,16 @@ def test_conftest_import_order(pytester: Pytester, monkeypatch: MonkeyPatch) -> ct2 = sub / "conftest.py" ct2.write_text("") - def impct(p, importmode): + def impct(p, importmode, root): return p conftest = PytestPluginManager() conftest._confcutdir = pytester.path monkeypatch.setattr(conftest, "_importconftest", impct) - mods = cast(List[Path], conftest._getconftestmodules(sub, importmode="prepend")) + mods = cast( + List[Path], + conftest._getconftestmodules(sub, importmode="prepend", rootpath=pytester.path), + ) expected = [ct1, ct2] assert mods == expected diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 69635e751fc..11db4c5c614 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -5,6 +5,7 @@ from pathlib import Path from textwrap import dedent from types import ModuleType +from typing import Any from typing import Generator import pytest @@ -17,7 +18,9 @@ from _pytest.pathlib import get_lock_path from _pytest.pathlib import import_path from _pytest.pathlib import ImportPathMismatchError +from _pytest.pathlib import insert_missing_modules from _pytest.pathlib import maybe_delete_a_numbered_dir +from _pytest.pathlib import module_name_from_path from _pytest.pathlib import resolve_package_path from _pytest.pathlib import symlink_or_skip from _pytest.pathlib import visit @@ -136,7 +139,7 @@ def setuptestfs(self, path: Path) -> None: ) def test_smoke_test(self, path1: Path) -> None: - obj = import_path(path1 / "execfile.py") + obj = import_path(path1 / "execfile.py", root=path1) assert obj.x == 42 # type: ignore[attr-defined] assert obj.__name__ == "execfile" @@ -146,25 +149,25 @@ def test_renamed_dir_creates_mismatch( tmp_path.joinpath("a").mkdir() p = tmp_path.joinpath("a", "test_x123.py") p.touch() - import_path(p) + import_path(p, root=tmp_path) tmp_path.joinpath("a").rename(tmp_path.joinpath("b")) with pytest.raises(ImportPathMismatchError): - import_path(tmp_path.joinpath("b", "test_x123.py")) + import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path) # Errors can be ignored. monkeypatch.setenv("PY_IGNORE_IMPORTMISMATCH", "1") - import_path(tmp_path.joinpath("b", "test_x123.py")) + import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path) # PY_IGNORE_IMPORTMISMATCH=0 does not ignore error. monkeypatch.setenv("PY_IGNORE_IMPORTMISMATCH", "0") with pytest.raises(ImportPathMismatchError): - import_path(tmp_path.joinpath("b", "test_x123.py")) + import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path) def test_messy_name(self, tmp_path: Path) -> None: # http://bitbucket.org/hpk42/py-trunk/issue/129 path = tmp_path / "foo__init__.py" path.touch() - module = import_path(path) + module = import_path(path, root=tmp_path) assert module.__name__ == "foo__init__" def test_dir(self, tmp_path: Path) -> None: @@ -172,31 +175,31 @@ def test_dir(self, tmp_path: Path) -> None: p.mkdir() p_init = p / "__init__.py" p_init.touch() - m = import_path(p) + m = import_path(p, root=tmp_path) assert m.__name__ == "hello_123" - m = import_path(p_init) + m = import_path(p_init, root=tmp_path) assert m.__name__ == "hello_123" def test_a(self, path1: Path) -> None: otherdir = path1 / "otherdir" - mod = import_path(otherdir / "a.py") + mod = import_path(otherdir / "a.py", root=path1) assert mod.result == "got it" # type: ignore[attr-defined] assert mod.__name__ == "otherdir.a" def test_b(self, path1: Path) -> None: otherdir = path1 / "otherdir" - mod = import_path(otherdir / "b.py") + mod = import_path(otherdir / "b.py", root=path1) assert mod.stuff == "got it" # type: ignore[attr-defined] assert mod.__name__ == "otherdir.b" def test_c(self, path1: Path) -> None: otherdir = path1 / "otherdir" - mod = import_path(otherdir / "c.py") + mod = import_path(otherdir / "c.py", root=path1) assert mod.value == "got it" # type: ignore[attr-defined] def test_d(self, path1: Path) -> None: otherdir = path1 / "otherdir" - mod = import_path(otherdir / "d.py") + mod = import_path(otherdir / "d.py", root=path1) assert mod.value2 == "got it" # type: ignore[attr-defined] def test_import_after(self, tmp_path: Path) -> None: @@ -204,7 +207,7 @@ def test_import_after(self, tmp_path: Path) -> None: tmp_path.joinpath("xxxpackage", "__init__.py").touch() mod1path = tmp_path.joinpath("xxxpackage", "module1.py") mod1path.touch() - mod1 = import_path(mod1path) + mod1 = import_path(mod1path, root=tmp_path) assert mod1.__name__ == "xxxpackage.module1" from xxxpackage import module1 @@ -222,7 +225,7 @@ def test_check_filepath_consistency( pseudopath.touch() mod.__file__ = str(pseudopath) monkeypatch.setitem(sys.modules, name, mod) - newmod = import_path(p) + newmod = import_path(p, root=tmp_path) assert mod == newmod monkeypatch.undo() mod = ModuleType(name) @@ -231,7 +234,7 @@ def test_check_filepath_consistency( mod.__file__ = str(pseudopath) monkeypatch.setitem(sys.modules, name, mod) with pytest.raises(ImportPathMismatchError) as excinfo: - import_path(p) + import_path(p, root=tmp_path) modname, modfile, orig = excinfo.value.args assert modname == name assert modfile == str(pseudopath) @@ -248,8 +251,8 @@ def test_issue131_on__init__(self, tmp_path: Path) -> None: tmp_path.joinpath("sub", "proja").mkdir(parents=True) p2 = tmp_path.joinpath("sub", "proja", "__init__.py") p2.touch() - m1 = import_path(p1) - m2 = import_path(p2) + m1 = import_path(p1, root=tmp_path) + m2 = import_path(p2, root=tmp_path) assert m1 == m2 def test_ensuresyspath_append(self, tmp_path: Path) -> None: @@ -258,44 +261,45 @@ def test_ensuresyspath_append(self, tmp_path: Path) -> None: file1 = root1 / "x123.py" file1.touch() assert str(root1) not in sys.path - import_path(file1, mode="append") + import_path(file1, mode="append", root=tmp_path) assert str(root1) == sys.path[-1] assert str(root1) not in sys.path[:-1] def test_invalid_path(self, tmp_path: Path) -> None: with pytest.raises(ImportError): - import_path(tmp_path / "invalid.py") + import_path(tmp_path / "invalid.py", root=tmp_path) @pytest.fixture def simple_module(self, tmp_path: Path) -> Path: - fn = tmp_path / "mymod.py" - fn.write_text( - dedent( - """ - def foo(x): return 40 + x - """ - ) - ) + fn = tmp_path / "_src/tests/mymod.py" + fn.parent.mkdir(parents=True) + fn.write_text("def foo(x): return 40 + x") return fn - def test_importmode_importlib(self, simple_module: Path) -> None: + def test_importmode_importlib(self, simple_module: Path, tmp_path: Path) -> None: """`importlib` mode does not change sys.path.""" - module = import_path(simple_module, mode="importlib") + module = import_path(simple_module, mode="importlib", root=tmp_path) assert module.foo(2) == 42 # type: ignore[attr-defined] assert str(simple_module.parent) not in sys.path + assert module.__name__ in sys.modules + assert module.__name__ == "_src.tests.mymod" + assert "_src" in sys.modules + assert "_src.tests" in sys.modules - def test_importmode_twice_is_different_module(self, simple_module: Path) -> None: + def test_importmode_twice_is_different_module( + self, simple_module: Path, tmp_path: Path + ) -> None: """`importlib` mode always returns a new module.""" - module1 = import_path(simple_module, mode="importlib") - module2 = import_path(simple_module, mode="importlib") + module1 = import_path(simple_module, mode="importlib", root=tmp_path) + module2 = import_path(simple_module, mode="importlib", root=tmp_path) assert module1 is not module2 def test_no_meta_path_found( - self, simple_module: Path, monkeypatch: MonkeyPatch + self, simple_module: Path, monkeypatch: MonkeyPatch, tmp_path: Path ) -> None: """Even without any meta_path should still import module.""" monkeypatch.setattr(sys, "meta_path", []) - module = import_path(simple_module, mode="importlib") + module = import_path(simple_module, mode="importlib", root=tmp_path) assert module.foo(2) == 42 # type: ignore[attr-defined] # mode='importlib' fails if no spec is found to load the module @@ -305,7 +309,7 @@ def test_no_meta_path_found( importlib.util, "spec_from_file_location", lambda *args: None ) with pytest.raises(ImportError): - import_path(simple_module, mode="importlib") + import_path(simple_module, mode="importlib", root=tmp_path) def test_resolve_package_path(tmp_path: Path) -> None: @@ -441,5 +445,130 @@ def test_samefile_false_negatives(tmp_path: Path, monkeypatch: MonkeyPatch) -> N # the paths too. Using a context to narrow the patch as much as possible given # this is an important system function. mp.setattr(os.path, "samefile", lambda x, y: False) - module = import_path(module_path) + module = import_path(module_path, root=tmp_path) assert getattr(module, "foo")() == 42 + + +class TestImportLibMode: + @pytest.mark.skipif(sys.version_info < (3, 7), reason="Dataclasses in Python3.7+") + def test_importmode_importlib_with_dataclass(self, tmp_path: Path) -> None: + """Ensure that importlib mode works with a module containing dataclasses (#7856).""" + fn = tmp_path.joinpath("_src/tests/test_dataclass.py") + fn.parent.mkdir(parents=True) + fn.write_text( + dedent( + """ + from dataclasses import dataclass + + @dataclass + class Data: + value: str + """ + ) + ) + + module = import_path(fn, mode="importlib", root=tmp_path) + Data: Any = getattr(module, "Data") + data = Data(value="foo") + assert data.value == "foo" + assert data.__module__ == "_src.tests.test_dataclass" + + def test_importmode_importlib_with_pickle(self, tmp_path: Path) -> None: + """Ensure that importlib mode works with pickle (#7859).""" + fn = tmp_path.joinpath("_src/tests/test_pickle.py") + fn.parent.mkdir(parents=True) + fn.write_text( + dedent( + """ + import pickle + + def _action(): + return 42 + + def round_trip(): + s = pickle.dumps(_action) + return pickle.loads(s) + """ + ) + ) + + module = import_path(fn, mode="importlib", root=tmp_path) + round_trip = getattr(module, "round_trip") + action = round_trip() + assert action() == 42 + + def test_importmode_importlib_with_pickle_separate_modules( + self, tmp_path: Path + ) -> None: + """ + Ensure that importlib mode works can load pickles that look similar but are + defined in separate modules. + """ + fn1 = tmp_path.joinpath("_src/m1/tests/test.py") + fn1.parent.mkdir(parents=True) + fn1.write_text( + dedent( + """ + import attr + import pickle + + @attr.s(auto_attribs=True) + class Data: + x: int = 42 + """ + ) + ) + + fn2 = tmp_path.joinpath("_src/m2/tests/test.py") + fn2.parent.mkdir(parents=True) + fn2.write_text( + dedent( + """ + import attr + import pickle + + @attr.s(auto_attribs=True) + class Data: + x: str = "" + """ + ) + ) + + import pickle + + def round_trip(obj): + s = pickle.dumps(obj) + return pickle.loads(s) + + module = import_path(fn1, mode="importlib", root=tmp_path) + Data1 = getattr(module, "Data") + + module = import_path(fn2, mode="importlib", root=tmp_path) + Data2 = getattr(module, "Data") + + assert round_trip(Data1(20)) == Data1(20) + assert round_trip(Data2("hello")) == Data2("hello") + assert Data1.__module__ == "_src.m1.tests.test" + assert Data2.__module__ == "_src.m2.tests.test" + + def test_module_name_from_path(self, tmp_path: Path) -> None: + result = module_name_from_path(tmp_path / "src/tests/test_foo.py", tmp_path) + assert result == "src.tests.test_foo" + + # Path is not relative to root dir: use the full path to obtain the module name. + result = module_name_from_path(Path("/home/foo/test_foo.py"), Path("/bar")) + assert result == "home.foo.test_foo" + + def test_insert_missing_modules(self) -> None: + modules = {"src.tests.foo": ModuleType("src.tests.foo")} + insert_missing_modules(modules, "src.tests.foo") + assert sorted(modules) == ["src", "src.tests", "src.tests.foo"] + + mod = ModuleType("mod", doc="My Module") + modules = {"src": mod} + insert_missing_modules(modules, "src") + assert modules == {"src": mod} + + modules = {} + insert_missing_modules(modules, "") + assert modules == {} diff --git a/testing/test_pluginmanager.py b/testing/test_pluginmanager.py index 9835b24a046..252591dd39a 100644 --- a/testing/test_pluginmanager.py +++ b/testing/test_pluginmanager.py @@ -44,7 +44,9 @@ def pytest_myhook(xyz): pm.hook.pytest_addhooks.call_historic( kwargs=dict(pluginmanager=config.pluginmanager) ) - config.pluginmanager._importconftest(conf, importmode="prepend") + config.pluginmanager._importconftest( + conf, importmode="prepend", rootpath=pytester.path + ) # print(config.pluginmanager.get_plugins()) res = config.hook.pytest_myhook(xyz=10) assert res == [11] @@ -71,7 +73,9 @@ def pytest_addoption(parser): default=True) """ ) - config.pluginmanager._importconftest(p, importmode="prepend") + config.pluginmanager._importconftest( + p, importmode="prepend", rootpath=pytester.path + ) assert config.option.test123 def test_configure(self, pytester: Pytester) -> None: @@ -136,10 +140,14 @@ def test_hook_proxy(self, pytester: Pytester) -> None: conftest1 = pytester.path.joinpath("tests/conftest.py") conftest2 = pytester.path.joinpath("tests/subdir/conftest.py") - config.pluginmanager._importconftest(conftest1, importmode="prepend") + config.pluginmanager._importconftest( + conftest1, importmode="prepend", rootpath=pytester.path + ) ihook_a = session.gethookproxy(pytester.path / "tests") assert ihook_a is not None - config.pluginmanager._importconftest(conftest2, importmode="prepend") + config.pluginmanager._importconftest( + conftest2, importmode="prepend", rootpath=pytester.path + ) ihook_b = session.gethookproxy(pytester.path / "tests") assert ihook_a is not ihook_b @@ -350,7 +358,9 @@ def test_consider_conftest_deps( pytester: Pytester, pytestpm: PytestPluginManager, ) -> None: - mod = import_path(pytester.makepyfile("pytest_plugins='xyz'")) + mod = import_path( + pytester.makepyfile("pytest_plugins='xyz'"), root=pytester.path + ) with pytest.raises(ImportError): pytestpm.consider_conftest(mod)