-
Notifications
You must be signed in to change notification settings - Fork 1.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Use importlib to load numba extensions #5209
Changes from 7 commits
2b672f2
bedf8da
4999274
22f4f04
22e91be
8f3c285
760ae11
c3d6990
44bcc9c
22e230a
86af993
0d03446
bbe8594
1fbb79f
6184d29
7992f28
169d106
e5fdc82
61ae870
c842eda
724990f
d94e66d
fbd66c3
b5f75f6
59fa60d
4456c37
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -80,6 +80,8 @@ $CONDA_INSTALL -c numba llvmlite | |||||
# Install enum34 and singledispatch for Python < 3.4 | ||||||
if [ $PYTHON \< "3.4" ]; then $CONDA_INSTALL enum34; fi | ||||||
if [ $PYTHON \< "3.4" ]; then $PIP_INSTALL singledispatch; fi | ||||||
# Install importlib-metadata for Python < 3.8 | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
if [ $PYTHON \< "3.8" ]; then $PIP_INSTALL importlib-metadata; fi | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Perhaps get this from the Anaconda distro?! |
||||||
# Install funcsigs for Python < 3.3 | ||||||
if [ $PYTHON \< "3.3" ]; then $CONDA_INSTALL -c numba funcsigs; fi | ||||||
# Install dependencies for building the documentation | ||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,24 +1,40 @@ | ||
import logging | ||
|
||
from pkg_resources import iter_entry_points | ||
from numba.core.config import PYVERSION | ||
|
||
if PYVERSION < (3, 8): | ||
try: | ||
import importlib_metadata | ||
except ImportError as ex: | ||
raise ImportError( | ||
"importlib_metadata backport is required for Python version < 3.8, " | ||
"try:\n" | ||
"$ conda/pip install importlib_metadata" | ||
) from ex | ||
else: | ||
from importlib import metadata as importlib_metadata | ||
|
||
|
||
_already_initialized = False | ||
logger = logging.getLogger(__name__) | ||
|
||
|
||
def init_all(): | ||
'''Execute all `numba_extensions` entry points with the name `init` | ||
"""Execute all `numba_extensions` entry points with the name `init` | ||
|
||
If extensions have already been initialized, this function does nothing. | ||
''' | ||
""" | ||
global _already_initialized | ||
if _already_initialized: | ||
return | ||
|
||
# Must put this here to avoid extensions re-triggering initialization | ||
_already_initialized = True | ||
|
||
for entry_point in iter_entry_points('numba_extensions', 'init'): | ||
logger.debug('Loading extension: %s', entry_point) | ||
func = entry_point.load() | ||
func() | ||
for entry_point in importlib_metadata.entry_points().get( | ||
"numba_extensions", tuple() | ||
): | ||
svrakitin marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if entry_point.name == "init": | ||
logger.debug("Loading extension: %s", entry_point) | ||
func = entry_point.load() | ||
func() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,10 @@ | ||
import sys | ||
import types | ||
from unittest import mock | ||
|
||
import pkg_resources | ||
try: | ||
from importlib import metadata as importlib_metadata | ||
except ImportError: | ||
import importlib_metadata | ||
|
||
from numba.tests.support import TestCase | ||
|
||
|
@@ -15,45 +18,38 @@ def test_init_entrypoint(self): | |
# loosely based on Pandas test from: | ||
# https://github.com/pandas-dev/pandas/pull/27488 | ||
|
||
# FIXME: Python 2 workaround because nonlocal doesn't exist | ||
counters = {'init': 0} | ||
|
||
def init_function(): | ||
counters['init'] += 1 | ||
|
||
mod = types.ModuleType("_test_numba_extension") | ||
mod.init_func = init_function | ||
mod = mock.Mock(__name__='_test_numba_extension') | ||
|
||
try: | ||
# will remove this module at the end of the test | ||
sys.modules[mod.__name__] = mod | ||
|
||
# We are registering an entry point using the "numba" package | ||
# ("distribution" in pkg_resources-speak) itself, though these are | ||
# ("distribution" in importlib-speak) itself, though these are | ||
# normally registered by other packages. | ||
stuartarchibald marked this conversation as resolved.
Show resolved
Hide resolved
|
||
dist = "numba" | ||
entrypoints = pkg_resources.get_entry_map(dist) | ||
my_entrypoint = pkg_resources.EntryPoint( | ||
"init", # name of entry point | ||
mod.__name__, # module with entry point object | ||
attrs=['init_func'], # name of entry point object | ||
dist=pkg_resources.get_distribution(dist) | ||
my_entrypoint = importlib_metadata.EntryPoint( | ||
'init', '_test_numba_extension:init_func', 'numba_extensions', | ||
) | ||
entrypoints.setdefault('numba_extensions', | ||
{})['init'] = my_entrypoint | ||
|
||
from numba.core import entrypoints | ||
# Allow reinitialization | ||
entrypoints._already_initialized = False | ||
with mock.patch.object( | ||
importlib_metadata, | ||
'entry_points', | ||
return_value={'numba_extensions': (my_entrypoint,)}, | ||
): | ||
|
||
from numba.core import entrypoints | ||
|
||
# Allow reinitialization | ||
entrypoints._already_initialized = False | ||
stuartarchibald marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
entrypoints.init_all() | ||
entrypoints.init_all() | ||
|
||
# was our init function called? | ||
self.assertEqual(counters['init'], 1) | ||
# was our init function called? | ||
mod.init_func.assert_called_once() | ||
Comment on lines
-68
to
+64
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Seems like a similar change can also be made to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Similar change made - note this also involves configuring the mock with a side effect for |
||
|
||
# ensure we do not initialize twice | ||
entrypoints.init_all() | ||
self.assertEqual(counters['init'], 1) | ||
# ensure we do not initialize twice | ||
entrypoints.init_all() | ||
mod.init_func.assert_called_once() | ||
finally: | ||
# remove fake module | ||
if mod.__name__ in sys.modules: | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Perhaps get this from the Anaconda distro?!