refactor: Split out the handler cache, expose it through the plugin

This makes `handlers_cache` no longer be global but instead be confined to the Plugin. There will be only one instance of the plugin so it doesn't matter anyway. But actually this is also more correct, because what if someone tried to instantiate multiple handlers with different configs? It would work incorrectly previously.

But my main goal for this is to expose `MkdocstringsPlugin.get_handler(name)`. Then someone can use this inside a mkdocs hook:

    def on_files(self, files: Files, config: Config):
        crystal = config['plugins']['mkdocstrings'].get_handler('python').collector

So this is basically a prerequisite for issue #179: one could query the collector to know which files to generate.

PR #191: #191

src/mkdocstrings/extension.py

@@ -35,7 +35,7 @@
 from markdown.extensions import Extension
 from markdown.util import AtomicString
 
-from mkdocstrings.handlers.base import CollectionError, get_handler
+from mkdocstrings.handlers.base import CollectionError, Handlers
 from mkdocstrings.loggers import get_logger
 from mkdocstrings.references import AutoRefInlineProcessor
 
@@ -95,7 +95,7 @@ class AutoDocProcessor(BlockProcessor):
     classname = "autodoc"
     regex = re.compile(r"^(?P<heading>#{1,6} *|)::: ?(?P<name>.+?) *$", flags=re.MULTILINE)
 
-    def __init__(self, parser: BlockParser, md: Markdown, config: dict) -> None:
+    def __init__(self, parser: BlockParser, md: Markdown, config: dict, handlers: Handlers) -> None:
         """
         Initialize the object.
 
@@ -104,10 +104,12 @@ def __init__(self, parser: BlockParser, md: Markdown, config: dict) -> None:
             md: A `markdown.Markdown` instance.
             config: The [configuration][mkdocstrings.plugin.MkdocstringsPlugin.config_scheme]
                 of the `mkdocstrings` plugin.
+            handlers: A [mkdocstrings.handlers.base.Handlers][] instance.
         """
         super().__init__(parser=parser)
         self.md = md
         self._config = config
+        self._handlers = handlers
 
     def test(self, parent: Element, block: Element) -> bool:
         """
@@ -180,16 +182,11 @@ def process_block(self, identifier: str, yaml_block: str, heading_level: int = 0
             A new XML element.
         """
         config = yaml.safe_load(yaml_block) or {}
-        handler_name = self.get_handler_name(config)
+        handler_name = self._handlers.get_handler_name(config)
 
         log.debug(f"Using handler '{handler_name}'")
-        handler_config = self.get_handler_config(handler_name)
-        handler = get_handler(
-            handler_name,
-            self._config["theme_name"],
-            self._config["mkdocstrings"]["custom_templates"],
-            **handler_config,
-        )
+        handler_config = self._handlers.get_handler_config(handler_name)
+        handler = self._handlers.get_handler(handler_name, handler_config)
 
         selection, rendering = get_item_configs(handler_config, config)
         if heading_level:
@@ -224,35 +221,6 @@ def process_block(self, identifier: str, yaml_block: str, heading_level: int = 0
 
         return atomic_brute_cast(xml_contents)  # type: ignore
 
-    def get_handler_name(self, config: dict) -> str:
-        """
-        Return the handler name defined in an "autodoc" instruction YAML configuration, or the global default handler.
-
-        Arguments:
-            config: A configuration dictionary, obtained from YAML below the "autodoc" instruction.
-
-        Returns:
-            The name of the handler to use.
-        """
-        if "handler" in config:
-            return config["handler"]
-        return self._config["mkdocstrings"]["default_handler"]
-
-    def get_handler_config(self, handler_name: str) -> dict:
-        """
-        Return the global configuration of the given handler.
-
-        Arguments:
-            handler_name: The name of the handler to get the global configuration of.
-
-        Returns:
-            The global configuration of the given handler. It can be an empty dictionary.
-        """
-        handlers = self._config["mkdocstrings"].get("handlers", {})
-        if handlers:
-            return handlers.get(handler_name, {})
-        return {}
-
 
 def get_item_configs(handler_config: dict, config: dict) -> Tuple[Mapping, Mapping]:
     """
@@ -307,17 +275,19 @@ class MkdocstringsExtension(Extension):
     blockprocessor_priority = 75  # Right before markdown.blockprocessors.HashHeaderProcessor
     inlineprocessor_priority = 168  # Right after markdown.inlinepatterns.ReferenceInlineProcessor
 
-    def __init__(self, config: dict, **kwargs) -> None:
+    def __init__(self, config: dict, handlers: Handlers, **kwargs) -> None:
         """
         Initialize the object.
 
         Arguments:
             config: The configuration items from `mkdocs` and `mkdocstrings` that must be passed to the block processor
                 when instantiated in [`extendMarkdown`][mkdocstrings.extension.MkdocstringsExtension.extendMarkdown].
+            handlers: A [mkdocstrings.handlers.base.Handlers][] instance.
             kwargs: Keyword arguments used by `markdown.extensions.Extension`.
         """
         super().__init__(**kwargs)
         self._config = config
+        self._handlers = handlers
 
     def extendMarkdown(self, md: Markdown) -> None:  # noqa: N802 (casing: parent method's name)
         """
@@ -329,7 +299,7 @@ def extendMarkdown(self, md: Markdown) -> None:  # noqa: N802 (casing: parent me
             md: A `markdown.Markdown` instance.
         """
         md.registerExtension(self)
-        processor = AutoDocProcessor(md.parser, md, self._config)
+        processor = AutoDocProcessor(md.parser, md, self._config, self._handlers)
         md.parser.blockprocessors.register(processor, "mkdocstrings", self.blockprocessor_priority)
         ref_processor = AutoRefInlineProcessor(md)
         md.inlinePatterns.register(ref_processor, "mkdocstrings", self.inlineprocessor_priority)

src/mkdocstrings/handlers/base.py

@@ -260,38 +260,85 @@ def __init__(self, collector: BaseCollector, renderer: BaseRenderer) -> None:
         self.renderer = renderer
 
 
-def get_handler(
-    name: str,
-    theme: str,
-    custom_templates: Optional[str] = None,
-    **config: Any,
-) -> BaseHandler:
+class Handlers:
     """
-    Get a handler thanks to its name.
+    A collection of handlers.
 
-    This function dynamically imports a module named "mkdocstrings.handlers.NAME", calls its
-    `get_handler` method to get an instance of a handler, and caches it in dictionary.
-    It means that during one run (for each reload when serving, or once when building),
-    a handler is instantiated only once, and reused for each "autodoc" instruction asking for it.
+    Do not instantiate this directly. [The plugin][mkdocstrings.plugin.MkdocstringsPlugin] will keep one instance of
+    this for the purpose of caching. Use [mkdocstrings.plugin.MkdocstringsPlugin.get_handler][] for convenient access.
+    """
 
-    Arguments:
-        name: The name of the handler. Really, it's the name of the Python module holding it.
-        theme: The name of the theme to use.
-        custom_templates: Directory containing custom templates.
-        config: Configuration passed to the handler.
+    def __init__(self, config: dict) -> None:
+        """
+        Initialize the object.
 
-    Returns:
-        An instance of a subclass of [`BaseHandler`][mkdocstrings.handlers.base.BaseHandler],
-        as instantiated by the `get_handler` method of the handler's module.
-    """
-    if name not in handlers_cache:
-        module = importlib.import_module(f"mkdocstrings.handlers.{name}")
-        handlers_cache[name] = module.get_handler(theme, custom_templates, **config)  # type: ignore
-    return handlers_cache[name]
+        Arguments:
+            config: Configuration options for `mkdocs` and `mkdocstrings`, read from `mkdocs.yml`. See the source code
+                of [mkdocstrings.plugin.MkdocstringsPlugin.on_config][] to see what's in this dictionary.
+        """
+        self._config = config
+        self._handlers: Dict[str, BaseHandler] = {}
+
+    def get_handler_name(self, config: dict) -> str:
+        """
+        Return the handler name defined in an "autodoc" instruction YAML configuration, or the global default handler.
+
+        Arguments:
+            config: A configuration dictionary, obtained from YAML below the "autodoc" instruction.
+
+        Returns:
+            The name of the handler to use.
+        """
+        config = self._config["mkdocstrings"]
+        if "handler" in config:
+            return config["handler"]
+        return config["default_handler"]
+
+    def get_handler_config(self, name: str) -> dict:
+        """
+        Return the global configuration of the given handler.
 
+        Arguments:
+            name: The name of the handler to get the global configuration of.
+
+        Returns:
+            The global configuration of the given handler. It can be an empty dictionary.
+        """
+        handlers = self._config["mkdocstrings"].get("handlers", {})
+        if handlers:
+            return handlers.get(name, {})
+        return {}
 
-def teardown() -> None:
-    """Teardown all cached handlers and clear the cache."""
-    for handler in handlers_cache.values():
-        handler.collector.teardown()
-    handlers_cache.clear()
+    def get_handler(self, name: str, handler_config: Optional[dict] = None) -> BaseHandler:
+        """
+        Get a handler thanks to its name.
+
+        This function dynamically imports a module named "mkdocstrings.handlers.NAME", calls its
+        `get_handler` method to get an instance of a handler, and caches it in dictionary.
+        It means that during one run (for each reload when serving, or once when building),
+        a handler is instantiated only once, and reused for each "autodoc" instruction asking for it.
+
+        Arguments:
+            name: The name of the handler. Really, it's the name of the Python module holding it.
+            handler_config: Configuration passed to the handler.
+
+        Returns:
+            An instance of a subclass of [`BaseHandler`][mkdocstrings.handlers.base.BaseHandler],
+            as instantiated by the `get_handler` method of the handler's module.
+        """
+        if name not in self._handlers:
+            if handler_config is None:
+                handler_config = self.get_handler_config(name)
+            module = importlib.import_module(f"mkdocstrings.handlers.{name}")
+            self._handlers[name] = module.get_handler(
+                self._config["theme_name"],
+                self._config["mkdocstrings"]["custom_templates"],
+                **handler_config,
+            )  # type: ignore
+        return self._handlers[name]
+
+    def teardown(self):
+        """Teardown all cached handlers and clear the cache."""
+        for handler in self._handlers.values():
+            handler.collector.teardown()
+        self._handlers.clear()

src/mkdocstrings/plugin.py

@@ -15,8 +15,8 @@
 and fixes them using the previously stored identifier-URL mapping.
 
 Once the documentation is built, the [`on_post_build` event hook](https://www.mkdocs.org/user-guide/plugins/#on_post_build)
-is triggered and calls the [`handlers.teardown()` method][mkdocstrings.handlers.base.teardown]. This method is used
-to teardown the handlers that were instantiated during documentation buildup.
+is triggered and calls the [`handlers.teardown()` method][mkdocstrings.handlers.base.Handlers.teardown]. This method is
+used to teardown the handlers that were instantiated during documentation buildup.
 
 Finally, when serving the documentation, it can add directories to watch
 during the [`on_serve` event hook](https://www.mkdocs.org/user-guide/plugins/#on_serve).
@@ -34,7 +34,7 @@
 from mkdocs.structure.toc import AnchorLink
 
 from mkdocstrings.extension import MkdocstringsExtension
-from mkdocstrings.handlers.base import teardown
+from mkdocstrings.handlers.base import BaseHandler, Handlers
 from mkdocstrings.loggers import get_logger
 from mkdocstrings.references import fix_refs
 
@@ -102,8 +102,8 @@ class MkdocstringsPlugin(BasePlugin):
     def __init__(self) -> None:
         """Initialize the object."""
         super().__init__()
-        self.mkdocstrings_extension: Optional[MkdocstringsExtension] = None
         self.url_map: Dict[Any, str] = {}
+        self.handlers: Optional[Handlers] = None
 
     def on_serve(self, server: Server, builder: Callable = None, **kwargs) -> Server:  # noqa: W0613 (unused arguments)
         """
@@ -164,8 +164,9 @@ def on_config(self, config: Config, **kwargs) -> Config:  # noqa: W0613 (unused
             "mkdocstrings": self.config,
         }
 
-        self.mkdocstrings_extension = MkdocstringsExtension(config=extension_config)
-        config["markdown_extensions"].append(self.mkdocstrings_extension)
+        self.handlers = Handlers(extension_config)
+        mkdocstrings_extension = MkdocstringsExtension(extension_config, self.handlers)
+        config["markdown_extensions"].append(mkdocstrings_extension)
         return config
 
     def on_page_content(self, html: str, page: Page, **kwargs) -> str:  # noqa: W0613 (unused arguments)
@@ -254,5 +255,23 @@ def on_post_build(self, **kwargs) -> None:  # noqa: W0613,R0201 (unused argument
         Arguments:
             kwargs: Additional arguments passed by MkDocs.
         """
-        log.debug("Tearing handlers down")
-        teardown()
+        if self.handlers:
+            log.debug("Tearing handlers down")
+            self.handlers.teardown()
+
+    def get_handler(self, handler_name: str) -> BaseHandler:
+        """
+        Get a handler by its name. See [mkdocstrings.handlers.base.Handlers.get_handler][].
+
+        Arguments:
+            handler_name: The name of the handler.
+
+        Raises:
+            RuntimeError: If the plugin hasn't been initialized with a config.
+
+        Returns:
+            An instance of a subclass of [`BaseHandler`][mkdocstrings.handlers.base.BaseHandler].
+        """
+        if not self.handlers:
+            raise RuntimeError("The plugin hasn't been initialized with a config yet")
+        return self.handlers.get_handler(handler_name)

tests/test_extension.py

@@ -2,6 +2,7 @@
 from markdown import Markdown
 
 from mkdocstrings.extension import MkdocstringsExtension
+from mkdocstrings.handlers.base import Handlers
 
 _DEFAULT_CONFIG = {  # noqa: WPS407 (mutable constant)
     "theme_name": "material",
@@ -13,14 +14,15 @@
 
 def test_render_html_escaped_sequences():
     """Assert HTML-escaped sequences are correctly parsed as XML."""
-    md = Markdown(extensions=[MkdocstringsExtension(_DEFAULT_CONFIG)])
+    config = _DEFAULT_CONFIG
+    md = Markdown(extensions=[MkdocstringsExtension(config, Handlers(config))])
     md.convert("::: tests.fixtures.html_escaped_sequences")
 
 
 def test_reference_inside_autodoc():
     """Assert cross-reference Markdown extension works correctly."""
     config = dict(_DEFAULT_CONFIG)
-    ext = MkdocstringsExtension(config)
+    ext = MkdocstringsExtension(config, Handlers(config))
     config["mdx"].append(ext)
 
     md = Markdown(extensions=[ext])

tests/test_references.py

@@ -3,6 +3,7 @@
 import pytest
 
 from mkdocstrings.extension import MkdocstringsExtension
+from mkdocstrings.handlers.base import Handlers
 from mkdocstrings.references import fix_refs, relative_url
 
 
@@ -54,7 +55,8 @@ def run_references_test(url_map, source, output, unmapped=None, from_url="page.h
         unmapped: The expected unmapped list.
         from_url: The source page URL.
     """
-    ext = MkdocstringsExtension({})
+    config = {}
+    ext = MkdocstringsExtension(config, Handlers(config))
     md = markdown.Markdown(extensions=[ext])
     content = md.convert(source)
     actual_output, actual_unmapped = fix_refs(content, from_url, url_map)