Implement __spec__ for files we run. #745 #838

CHANGES.rst

@@ -24,10 +24,17 @@ want to know what's different in 5.0 since 4.5.x, see :ref:`whatsnew5x`.
 Unreleased
 ----------
 
+- Python files run with ``-m`` now have ``__spec__`` defined properly.  This
+  fixes `issue 745`_ (about not being able to run unittest tests that spawn
+  subprocesses), and `issue 838`_, which described the problem directly.
+
 - The :func:`.coverage.numbits.register_sqlite_functions` function now also
   registers `numbits_to_nums` for use in SQLite queries.  Thanks, Simon
   Willison.
 
+.. _issue 745: https://github.com/nedbat/coveragepy/issues/745
+.. _issue 838: https://github.com/nedbat/coveragepy/issues/838
+
 
 .. _changes_50b1:
 

coverage/execfile.py

@@ -33,8 +33,8 @@ def __init__(self, fullname, *_args):
     def find_module(modulename):
         """Find the module named `modulename`.
 
-        Returns the file path of the module, and the name of the enclosing
-        package.
+        Returns the file path of the module, the name of the enclosing
+        package, and the spec.
         """
         try:
             spec = importlib_util_find_spec(modulename)
@@ -44,7 +44,7 @@ def find_module(modulename):
             raise NoSource("No module named %r" % (modulename,))
         pathname = spec.origin
         packagename = spec.name
-        if pathname.endswith("__init__.py") and not modulename.endswith("__init__"):
+        if spec.submodule_search_locations:
             mod_main = modulename + ".__main__"
             spec = importlib_util_find_spec(mod_main)
             if not spec:
@@ -56,13 +56,13 @@ def find_module(modulename):
             pathname = spec.origin
             packagename = spec.name
         packagename = packagename.rpartition(".")[0]
-        return pathname, packagename
+        return pathname, packagename, spec
 else:
     def find_module(modulename):
         """Find the module named `modulename`.
 
-        Returns the file path of the module, and the name of the enclosing
-        package.
+        Returns the file path of the module, the name of the enclosing
+        package, and None (where a spec would have been).
         """
         openfile = None
         glo, loc = globals(), locals()
@@ -98,7 +98,7 @@ def find_module(modulename):
             if openfile:
                 openfile.close()
 
-        return pathname, packagename
+        return pathname, packagename, None
 
 
 class PyRunner(object):
@@ -112,7 +112,7 @@ def __init__(self, args, as_module=False):
         self.as_module = as_module
 
         self.arg0 = args[0]
-        self.package = self.modulename = self.pathname = None
+        self.package = self.modulename = self.pathname = self.loader = self.spec = None
 
     def prepare(self):
         """Do initial preparation to run Python code.
@@ -131,7 +131,10 @@ def prepare(self):
             sys.path[0] = path0
             should_update_sys_path = False
             self.modulename = self.arg0
-            pathname, self.package = find_module(self.modulename)
+            pathname, self.package, self.spec = find_module(self.modulename)
+            if self.spec is not None:
+                self.modulename = self.spec.name
+            self.loader = DummyLoader(self.modulename)
             self.pathname = os.path.abspath(pathname)
             self.args[0] = self.arg0 = self.pathname
         elif os.path.isdir(self.arg0):
@@ -145,10 +148,26 @@ def prepare(self):
                     break
             else:
                 raise NoSource("Can't find '__main__' module in '%s'" % self.arg0)
+
+            if env.PY2:
+                self.arg0 = os.path.abspath(self.arg0)
+
+            # Make a spec. I don't know if this is the right way to do it.
+            try:
+                import importlib.machinery
+            except ImportError:
+                pass
+            else:
+                self.spec = importlib.machinery.ModuleSpec("__main__", None, origin=try_filename)
+                self.spec.has_location = True
+            self.package = ""
+            self.loader = DummyLoader("__main__")
         else:
             path0 = os.path.abspath(os.path.dirname(self.arg0))
+            if env.PY3:
+                self.loader = DummyLoader("__main__")
 
-        if self.modulename is None and env.PYVERSION >= (3, 3):
+        if self.modulename is None:
             self.modulename = '__main__'
 
         if should_update_sys_path:
@@ -167,21 +186,27 @@ def run(self):
 
         # Create a module to serve as __main__
         main_mod = types.ModuleType('__main__')
-        sys.modules['__main__'] = main_mod
+
+        from_pyc = self.arg0.endswith((".pyc", ".pyo"))
         main_mod.__file__ = self.arg0
-        if self.package:
+        if from_pyc:
+            main_mod.__file__ = main_mod.__file__[:-1]
+        if self.package is not None:
             main_mod.__package__ = self.package
-        if self.modulename:
-            main_mod.__loader__ = DummyLoader(self.modulename)
+        main_mod.__loader__ = self.loader
+        if self.spec is not None:
+            main_mod.__spec__ = self.spec
 
         main_mod.__builtins__ = BUILTINS
 
+        sys.modules['__main__'] = main_mod
+
         # Set sys.argv properly.
         sys.argv = self.args
 
         try:
             # Make a code object somehow.
-            if self.arg0.endswith((".pyc", ".pyo")):
+            if from_pyc:
                 code = make_code_from_pyc(self.arg0)
             else:
                 code = make_code_from_py(self.arg0)

tests/modules/process_test/try_execfile.py

@@ -66,7 +66,7 @@ def my_function(a):
 FN_VAL = my_function("fooey")
 
 loader = globals().get('__loader__')
-fullname = getattr(loader, 'fullname', None) or getattr(loader, 'name', None)
+spec = globals().get('__spec__')
 
 # A more compact ad-hoc grouped-by-first-letter list of builtins.
 CLUMPS = "ABC,DEF,GHI,JKLMN,OPQR,ST,U,VWXYZ_,ab,cd,efg,hij,lmno,pqr,stuvwxyz".split(",")
@@ -88,8 +88,8 @@ def word_group(w):
     '__builtins__.has_open': hasattr(__builtins__, 'open'),
     '__builtins__.dir': builtin_dir,
     '__loader__ exists': loader is not None,
-    '__loader__.fullname': fullname,
     '__package__': __package__,
+    '__spec__ exists': spec is not None,
     'DATA': DATA,
     'FN_VAL': FN_VAL,
     '__main__.DATA': getattr(__main__, "DATA", "nothing"),
@@ -98,4 +98,15 @@ def word_group(w):
     'path': cleaned_sys_path,
 }
 
+if loader is not None:
+    globals_to_check.update({
+        '__loader__.fullname': getattr(loader, 'fullname', None) or getattr(loader, 'name', None)
+    })
+
+if spec is not None:
+    globals_to_check.update({
+        '__spec__.' + aname: getattr(spec, aname)
+        for aname in ['name', 'origin', 'submodule_search_locations', 'parent', 'has_location']
+    })
+
 print(json.dumps(globals_to_check, indent=4, sort_keys=True))

tests/test_process.py

@@ -866,17 +866,36 @@ def test_coverage_run_dir_is_like_python_dir(self):
         expected = self.run_command("python with_main")
         actual = self.run_command("coverage run with_main")
 
-        # The coverage.py results are not identical to the Python results, and
-        # I don't know why.  For now, ignore those failures. If someone finds
-        # a real problem with the discrepancies, we can work on it some more.
-        ignored = r"__file__|__loader__|__package__"
         # PyPy includes the current directory in the path when running a
         # directory, while CPython and coverage.py do not.  Exclude that from
         # the comparison also...
         if env.PYPY:
-            ignored += "|"+re.escape(os.getcwd())
-        expected = re_lines(expected, ignored, match=False)
-        actual = re_lines(actual, ignored, match=False)
+            ignored = re.escape(os.getcwd())
+            expected = re_lines(expected, ignored, match=False)
+            actual = re_lines(actual, ignored, match=False)
+        self.assert_tryexecfile_output(expected, actual)
+
+    def test_coverage_run_dashm_dir_no_init_is_like_python(self):
+        with open(TRY_EXECFILE) as f:
+            self.make_file("with_main/__main__.py", f.read())
+
+        expected = self.run_command("python -m with_main")
+        actual = self.run_command("coverage run -m with_main")
+        if env.PY2:
+            assert expected.endswith("No module named with_main\n")
+            assert actual.endswith("No module named with_main\n")
+        else:
+            self.assert_tryexecfile_output(expected, actual)
+
+    def test_coverage_run_dashm_dir_with_init_is_like_python(self):
+        if env.PY2:
+            self.skipTest("Python 2 runs __main__ twice, I can't be bothered to make it work.")
+        with open(TRY_EXECFILE) as f:
+            self.make_file("with_main/__main__.py", f.read())
+        self.make_file("with_main/__init__.py", "")
+
+        expected = self.run_command("python -m with_main")
+        actual = self.run_command("coverage run -m with_main")
         self.assert_tryexecfile_output(expected, actual)
 
     def test_coverage_run_dashm_equal_to_doubledashsource(self):