-
Notifications
You must be signed in to change notification settings - Fork 1.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Use importlib to load numba extensions #5209
Changes from 4 commits
2b672f2
bedf8da
4999274
22f4f04
22e91be
8f3c285
760ae11
c3d6990
44bcc9c
22e230a
86af993
0d03446
bbe8594
1fbb79f
6184d29
7992f28
169d106
e5fdc82
61ae870
c842eda
724990f
d94e66d
fbd66c3
b5f75f6
59fa60d
4456c37
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -80,6 +80,8 @@ $CONDA_INSTALL -c numba llvmlite | |||||
# Install enum34 and singledispatch for Python < 3.4 | ||||||
if [ $PYTHON \< "3.4" ]; then $CONDA_INSTALL enum34; fi | ||||||
if [ $PYTHON \< "3.4" ]; then $PIP_INSTALL singledispatch; fi | ||||||
# Install importlib-metadata for Python < 3.8 | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
if [ $PYTHON \< "3.8" ]; then $PIP_INSTALL importlib-metadata; fi | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Perhaps get this from the Anaconda distro?! |
||||||
# Install funcsigs for Python < 3.3 | ||||||
if [ $PYTHON \< "3.3" ]; then $CONDA_INSTALL -c numba funcsigs; fi | ||||||
# Install dependencies for building the documentation | ||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,9 @@ | ||
import logging | ||
|
||
from pkg_resources import iter_entry_points | ||
try: | ||
from importlib import metadata as importlib_metadata | ||
except ImportError: | ||
import importlib_metadata | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What if this fails? I'm wondering if there ought to be an explicit Python>=3.8 branch which just imports it and then another which tries and fails gracefully if the package is missing. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It will only fail on Python < 3.8 if importlib_metadata is missing (it is listed as requirement). There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We tend to assume nothing about how Numba was built, it's entirely possible someone just grabbed the source and some limited set of bootstrap packages available for their unusual architecture and built it (this actually happens in practice, we see these people on the issue tracker when strange things happen!). As a result, we tend to err on the side of trying to be as helpful as possible and catch things like this and say something about it being needed. Further, this will need adding to the dependencies section in the docs: https://github.com/numba/numba/blob/13ece9b97e6f01f750e870347f231282325f60c3/docs/source/user/installing.rst#dependency-list |
||
|
||
_already_initialized = False | ||
logger = logging.getLogger(__name__) | ||
|
@@ -18,7 +21,10 @@ def init_all(): | |
# Must put this here to avoid extensions re-triggering initialization | ||
_already_initialized = True | ||
|
||
for entry_point in iter_entry_points('numba_extensions', 'init'): | ||
logger.debug('Loading extension: %s', entry_point) | ||
func = entry_point.load() | ||
func() | ||
for entry_point in importlib_metadata.entry_points().get( | ||
'numba_extensions', tuple() | ||
): | ||
if entry_point.name == 'init': | ||
logger.debug('Loading extension: %s', entry_point) | ||
func = entry_point.load() | ||
func() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,10 @@ | ||
import sys | ||
import types | ||
from unittest import mock | ||
|
||
import pkg_resources | ||
try: | ||
from importlib import metadata as importlib_metadata | ||
except ImportError: | ||
import importlib_metadata | ||
|
||
from numba.tests.support import TestCase | ||
|
||
|
@@ -15,45 +18,38 @@ def test_init_entrypoint(self): | |
# loosely based on Pandas test from: | ||
# https://github.com/pandas-dev/pandas/pull/27488 | ||
|
||
# FIXME: Python 2 workaround because nonlocal doesn't exist | ||
counters = {'init': 0} | ||
|
||
def init_function(): | ||
counters['init'] += 1 | ||
|
||
mod = types.ModuleType("_test_numba_extension") | ||
mod.init_func = init_function | ||
mod = mock.Mock(__name__='_test_numba_extension') | ||
|
||
try: | ||
# will remove this module at the end of the test | ||
sys.modules[mod.__name__] = mod | ||
|
||
# We are registering an entry point using the "numba" package | ||
# ("distribution" in pkg_resources-speak) itself, though these are | ||
# ("distribution" in importlib-speak) itself, though these are | ||
# normally registered by other packages. | ||
stuartarchibald marked this conversation as resolved.
Show resolved
Hide resolved
|
||
dist = "numba" | ||
entrypoints = pkg_resources.get_entry_map(dist) | ||
my_entrypoint = pkg_resources.EntryPoint( | ||
"init", # name of entry point | ||
mod.__name__, # module with entry point object | ||
attrs=['init_func'], # name of entry point object | ||
dist=pkg_resources.get_distribution(dist) | ||
my_entrypoint = importlib_metadata.EntryPoint( | ||
'init', '_test_numba_extension:init_func', 'numba_extensions', | ||
) | ||
entrypoints.setdefault('numba_extensions', | ||
{})['init'] = my_entrypoint | ||
|
||
from numba.core import entrypoints | ||
# Allow reinitialization | ||
entrypoints._already_initialized = False | ||
with mock.patch.object( | ||
importlib_metadata, | ||
'entry_points', | ||
return_value={'numba_extensions': (my_entrypoint,)}, | ||
): | ||
|
||
from numba.core import entrypoints | ||
|
||
# Allow reinitialization | ||
entrypoints._already_initialized = False | ||
stuartarchibald marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
entrypoints.init_all() | ||
entrypoints.init_all() | ||
|
||
# was our init function called? | ||
self.assertEqual(counters['init'], 1) | ||
# was our init function called? | ||
mod.init_func.assert_called_once() | ||
Comment on lines
-68
to
+64
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Seems like a similar change can also be made to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Similar change made - note this also involves configuring the mock with a side effect for |
||
|
||
# ensure we do not initialize twice | ||
entrypoints.init_all() | ||
self.assertEqual(counters['init'], 1) | ||
# ensure we do not initialize twice | ||
entrypoints.init_all() | ||
mod.init_func.assert_called_once() | ||
finally: | ||
# remove fake module | ||
if mod.__name__ in sys.modules: | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Perhaps get this from the Anaconda distro?!