From 0ab14a07cc1334466716ba2dc364d29255853da6 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet Date: Sun, 7 Apr 2024 17:24:59 +0200 Subject: [PATCH 01/27] WIP: New i18n logic based on inline_parse function Allows to not rely on strange hacks that are RST dependant. There is still an issue With the warning of missing literal block --- sphinx/io.py | 6 ++ sphinx/parsers.py | 28 ++++++++ sphinx/transforms/i18n.py | 67 ++++--------------- sphinx/util/nodes.py | 3 +- .../roots/test-intl/xx/LC_MESSAGES/figure.po | 8 +-- .../LC_MESSAGES/prolog_epilog_substitution.po | 8 +-- 6 files changed, 55 insertions(+), 65 deletions(-) diff --git a/sphinx/io.py b/sphinx/io.py index 459d250e45f..067aa917fab 100644 --- a/sphinx/io.py +++ b/sphinx/io.py @@ -143,6 +143,12 @@ def setup(self, app: Sphinx) -> None: if transform in self.transforms: self.transforms.remove(transform) + def parse(self): + """Parse `self.input` into a document tree.""" + self.document = document = self.new_document() + self.parser.parse_inline(self.input, document) + document.current_source = document.current_line = None + class SphinxDummyWriter(UnfilteredWriter): """Dummy writer module used for generating doctree.""" diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 955d59b3b79..99a0ed205ca 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -46,6 +46,10 @@ def set_application(self, app: Sphinx) -> None: self.config = app.config self.env = app.env + def parse_inline(self, inputstring: str, lineno: int) -> nodes.Node: + """Parse the inline elements of a text block and generate a document tree.""" + raise NotImplementedError('Parser subclasses must implement parse_inline') + class RSTParser(docutils.parsers.rst.Parser, Parser): """A reST parser for Sphinx.""" @@ -60,6 +64,30 @@ def get_transforms(self) -> list[type[Transform]]: transforms.remove(SmartQuotes) return transforms + def parse_inline(self, inputstring: str | StringList, document: nodes.document) -> None: + """Parse inline syntax from text and generate a document tree.""" + self.setup_parse(inputstring, document) # type: ignore[arg-type] + self.statemachine = states.RSTStateMachine( + state_classes=self.state_classes, + initial_state='Text', + debug=document.reporter.debug_flag, + ) + + # preprocess inputstring + if isinstance(inputstring, str): + lines = docutils.statemachine.string2lines( + inputstring, tab_width=document.settings.tab_width, convert_whitespace=True + ) + + inputlines = StringList(lines, document.current_source) + else: + inputlines = inputstring + + self.decorate(inputlines) + self.statemachine.run(inputlines, document, inliner=self.inliner) + self.finish_parse() + + def parse(self, inputstring: str | StringList, document: nodes.document) -> None: """Parse text and generate a document tree.""" self.setup_parse(inputstring, document) # type: ignore[arg-type] diff --git a/sphinx/transforms/i18n.py b/sphinx/transforms/i18n.py index 4f8c3530d75..0530a6469fb 100644 --- a/sphinx/transforms/i18n.py +++ b/sphinx/transforms/i18n.py @@ -380,25 +380,12 @@ def apply(self, **kwargs: Any) -> None: node['translated'] = True continue - # Avoid "Literal block expected; none found." warnings. - # If msgstr ends with '::' then it cause warning message at - # parser.parse() processing. - # literal-block-warning is only appear in avobe case. - if msgstr.strip().endswith('::'): - msgstr += '\n\n dummy literal' - # dummy literal node will discard by 'patch = patch[0]' - - # literalblock need literal block notation to avoid it become - # paragraph. + # literalblock can not contain references or terms if isinstance(node, LITERAL_TYPE_NODES): - msgstr = '::\n\n' + indent(msgstr, ' ' * 3) + continue patch = publish_msgstr(self.app, msgstr, source, node.line, self.config, settings) # type: ignore[arg-type] - # FIXME: no warnings about inconsistent references in this part - # XXX doctest and other block markup - if not isinstance(patch, nodes.paragraph): - continue # skip for now updater = _NodeUpdater(node, patch, self.document, noqa=False) processed = updater.update_title_mapping() @@ -453,45 +440,20 @@ def apply(self, **kwargs: Any) -> None: node['alt'] = msgstr continue - # Avoid "Literal block expected; none found." warnings. - # If msgstr ends with '::' then it cause warning message at - # parser.parse() processing. - # literal-block-warning is only appear in avobe case. - if msgstr.strip().endswith('::'): - msgstr += '\n\n dummy literal' - # dummy literal node will discard by 'patch = patch[0]' - - # literalblock need literal block notation to avoid it become - # paragraph. - if isinstance(node, LITERAL_TYPE_NODES): - msgstr = '::\n\n' + indent(msgstr, ' ' * 3) + if isinstance(node, nodes.image) and node.get('uri') == msg: + node['uri'] = msgstr + continue - # Structural Subelements phase1 - # There is a possibility that only the title node is created. - # see: https://docutils.sourceforge.io/docs/ref/doctree.html#structural-subelements - if isinstance(node, nodes.title): - # This generates:
msgstr
- msgstr = msgstr + '\n' + '=' * len(msgstr) * 2 + # literalblock and images do not need to be parsed as they do not contain inline syntax + if isinstance(node, LITERAL_TYPE_NODES + IMAGE_TYPE_NODES): + patch = nodes.paragraph(text=msgstr) + else: + patch = publish_msgstr(self.app, msgstr, source, + node.line, self.config, settings) # type: ignore[arg-type] - patch = publish_msgstr(self.app, msgstr, source, - node.line, self.config, settings) # type: ignore[arg-type] # Structural Subelements phase2 - if isinstance(node, nodes.title): - # get node that placed as a first child - patch = patch.next_node() # type: ignore[assignment] - # ignore unexpected markups in translation message - unexpected: tuple[type[nodes.Element], ...] = ( - nodes.paragraph, # expected form of translation - nodes.title, # generated by above "Subelements phase2" - ) - - # following types are expected if - # config.gettext_additional_targets is configured - unexpected += LITERAL_TYPE_NODES - unexpected += IMAGE_TYPE_NODES - - if not isinstance(patch, unexpected): + if not isinstance(patch, nodes.paragraph): continue # skip updater = _NodeUpdater(node, patch, self.document, noqa) @@ -506,11 +468,6 @@ def apply(self, **kwargs: Any) -> None: if isinstance(node, LITERAL_TYPE_NODES): node.rawsource = node.astext() - if isinstance(node, nodes.image) and node.get('alt') != msg: - node['uri'] = patch['uri'] - node['translated'] = False - continue # do not mark translated - node['translated'] = True # to avoid double translation if 'index' in self.config.gettext_additional_targets: diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index bbc1f64e481..528f1729bef 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -266,8 +266,7 @@ def extract_messages(doctree: Element) -> Iterable[tuple[Element, str]]: if node.get('alt'): yield node, node['alt'] if node.get('translatable'): - image_uri = node.get('original_uri', node['uri']) - msg = f'.. image:: {image_uri}' + msg = node.get('original_uri', node['uri']) else: msg = '' elif isinstance(node, nodes.meta): diff --git a/tests/roots/test-intl/xx/LC_MESSAGES/figure.po b/tests/roots/test-intl/xx/LC_MESSAGES/figure.po index 64bbdf763db..4678aaf69a5 100644 --- a/tests/roots/test-intl/xx/LC_MESSAGES/figure.po +++ b/tests/roots/test-intl/xx/LC_MESSAGES/figure.po @@ -40,14 +40,14 @@ msgstr "IMAGE URL AND ALT" msgid "img" msgstr "IMG -> I18N" -msgid ".. image:: img.png" -msgstr ".. image:: i18n.png" +msgid "img.png" +msgstr "i18n.png" msgid "i18n" msgstr "I18N -> IMG" -msgid ".. image:: i18n.png" -msgstr ".. image:: img.png" +msgid "i18n.png" +msgstr "img.png" msgid "image on substitution" msgstr "IMAGE ON SUBSTITUTION" diff --git a/tests/roots/test-intl_substitution_definitions/xx/LC_MESSAGES/prolog_epilog_substitution.po b/tests/roots/test-intl_substitution_definitions/xx/LC_MESSAGES/prolog_epilog_substitution.po index 3ce51fe4ffc..7b32be8da7e 100644 --- a/tests/roots/test-intl_substitution_definitions/xx/LC_MESSAGES/prolog_epilog_substitution.po +++ b/tests/roots/test-intl_substitution_definitions/xx/LC_MESSAGES/prolog_epilog_substitution.po @@ -28,11 +28,11 @@ msgstr "SUBSTITUTED IMAGE |subst_epilog_2| HERE." msgid "subst_prolog_2" msgstr "SUBST_PROLOG_2 TRANSLATED" -msgid ".. image:: /img.png" -msgstr ".. image:: /i18n.png" +msgid "/img.png" +msgstr "/i18n.png" msgid "subst_epilog_2" msgstr "SUBST_EPILOG_2 TRANSLATED" -msgid ".. image:: /i18n.png" -msgstr ".. image:: /img.png" +msgid "/i18n.png" +msgstr "/img.png" From 2682f8b8ab933af221596aacb917829e43118db9 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 7 Apr 2024 17:55:17 +0200 Subject: [PATCH 02/27] Avoid 'Literal block expected; none found.' warnings --- sphinx/parsers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 99a0ed205ca..dde4ad24e7b 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -66,6 +66,12 @@ def get_transforms(self) -> list[type[Transform]]: def parse_inline(self, inputstring: str | StringList, document: nodes.document) -> None: """Parse inline syntax from text and generate a document tree.""" + + # Avoid "Literal block expected; none found." warnings. + if inputstring.endswith('::'): + inputstring += '\n\n dummy literal' + # dummy literal node will discard by 'patch = patch[0]' + self.setup_parse(inputstring, document) # type: ignore[arg-type] self.statemachine = states.RSTStateMachine( state_classes=self.state_classes, From 38c004bc56de3a2a2f8b63ebfc0b81364d79089b Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 7 Apr 2024 18:27:44 +0200 Subject: [PATCH 03/27] Update gettext builder output in tests --- tests/test_builders/test_build_gettext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_builders/test_build_gettext.py b/tests/test_builders/test_build_gettext.py index ddc6d30d4e4..802af985da5 100644 --- a/tests/test_builders/test_build_gettext.py +++ b/tests/test_builders/test_build_gettext.py @@ -207,11 +207,11 @@ def test_gettext_prolog_epilog_substitution(app): "This is content that contains |subst_prolog_1|.", "Substituted image |subst_prolog_2| here.", "subst_prolog_2", - ".. image:: /img.png", + "/img.png", "This is content that contains |subst_epilog_1|.", "Substituted image |subst_epilog_2| here.", "subst_epilog_2", - ".. image:: /i18n.png", + "/i18n.png", ] From 9a2510a3f2b4efd7637e477bf55c3841bd494e44 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 7 Apr 2024 19:58:35 +0200 Subject: [PATCH 04/27] parse_inline's input string is always a string --- sphinx/parsers.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index dde4ad24e7b..d2832b1a17a 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -46,7 +46,7 @@ def set_application(self, app: Sphinx) -> None: self.config = app.config self.env = app.env - def parse_inline(self, inputstring: str, lineno: int) -> nodes.Node: + def parse_inline(self, inputstring: str, document: nodes.document) -> None: """Parse the inline elements of a text block and generate a document tree.""" raise NotImplementedError('Parser subclasses must implement parse_inline') @@ -64,7 +64,7 @@ def get_transforms(self) -> list[type[Transform]]: transforms.remove(SmartQuotes) return transforms - def parse_inline(self, inputstring: str | StringList, document: nodes.document) -> None: + def parse_inline(self, inputstring: str, document: nodes.document) -> None: """Parse inline syntax from text and generate a document tree.""" # Avoid "Literal block expected; none found." warnings. @@ -79,15 +79,7 @@ def parse_inline(self, inputstring: str | StringList, document: nodes.document) debug=document.reporter.debug_flag, ) - # preprocess inputstring - if isinstance(inputstring, str): - lines = docutils.statemachine.string2lines( - inputstring, tab_width=document.settings.tab_width, convert_whitespace=True - ) - - inputlines = StringList(lines, document.current_source) - else: - inputlines = inputstring + inputlines = StringList([inputstring], document.current_source) self.decorate(inputlines) self.statemachine.run(inputlines, document, inliner=self.inliner) From 3c97944070640d9ee7c413f898d7220d20ea0ff8 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 7 Apr 2024 20:00:12 +0200 Subject: [PATCH 05/27] Only output a single paragraph in RSTParser's parse_inline So we trim the literal suffix to avoid warnings and we add it back at the end --- sphinx/parsers.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index d2832b1a17a..6c6d82b2910 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -68,9 +68,9 @@ def parse_inline(self, inputstring: str, document: nodes.document) -> None: """Parse inline syntax from text and generate a document tree.""" # Avoid "Literal block expected; none found." warnings. - if inputstring.endswith('::'): - inputstring += '\n\n dummy literal' - # dummy literal node will discard by 'patch = patch[0]' + has_literal = inputstring.endswith('::') + if has_literal: + inputstring = inputstring[:-2] self.setup_parse(inputstring, document) # type: ignore[arg-type] self.statemachine = states.RSTStateMachine( @@ -84,6 +84,10 @@ def parse_inline(self, inputstring: str, document: nodes.document) -> None: self.decorate(inputlines) self.statemachine.run(inputlines, document, inliner=self.inliner) self.finish_parse() + if has_literal: + p = document[0] + assert(isinstance(p, nodes.paragraph)) + p += nodes.Text(':') def parse(self, inputstring: str | StringList, document: nodes.document) -> None: From 66b14d3f216e25886d0e2b6103f1cca8cd84499d Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 7 Apr 2024 20:19:28 +0200 Subject: [PATCH 06/27] Image nodes are handled separately, and literal can simply be manually patched --- sphinx/transforms/i18n.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/sphinx/transforms/i18n.py b/sphinx/transforms/i18n.py index 0530a6469fb..70dda14cfff 100644 --- a/sphinx/transforms/i18n.py +++ b/sphinx/transforms/i18n.py @@ -21,7 +21,6 @@ from sphinx.util.i18n import docname_to_domain from sphinx.util.index_entries import split_index_msg from sphinx.util.nodes import ( - IMAGE_TYPE_NODES, LITERAL_TYPE_NODES, NodeMatcher, extract_messages, @@ -444,14 +443,17 @@ def apply(self, **kwargs: Any) -> None: node['uri'] = msgstr continue - # literalblock and images do not need to be parsed as they do not contain inline syntax - if isinstance(node, LITERAL_TYPE_NODES + IMAGE_TYPE_NODES): - patch = nodes.paragraph(text=msgstr) - else: - patch = publish_msgstr(self.app, msgstr, source, - node.line, self.config, settings) # type: ignore[arg-type] + # literalblock do not need to be parsed as they do not contain inline syntax + if isinstance(node, LITERAL_TYPE_NODES): + node.children = [nodes.Text(msgstr)] + # for highlighting that expects .rawsource and .astext() are same. + node.rawsource = node.astext() + node['translated'] = True + continue + + patch = publish_msgstr(self.app, msgstr, source, + node.line, self.config, settings) # type: ignore[arg-type] - # Structural Subelements phase2 # ignore unexpected markups in translation message if not isinstance(patch, nodes.paragraph): continue # skip @@ -464,10 +466,6 @@ def apply(self, **kwargs: Any) -> None: updater.update_pending_xrefs() updater.update_leaves() - # for highlighting that expects .rawsource and .astext() are same. - if isinstance(node, LITERAL_TYPE_NODES): - node.rawsource = node.astext() - node['translated'] = True # to avoid double translation if 'index' in self.config.gettext_additional_targets: From 5994ca5594d7ce719c49c76d3a707e88c0a563b5 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 7 Apr 2024 20:48:02 +0200 Subject: [PATCH 07/27] Fix ruff lints + more useful comment --- sphinx/io.py | 4 ++-- sphinx/parsers.py | 4 +--- sphinx/transforms/i18n.py | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/sphinx/io.py b/sphinx/io.py index 067aa917fab..4fdcca09a04 100644 --- a/sphinx/io.py +++ b/sphinx/io.py @@ -143,8 +143,8 @@ def setup(self, app: Sphinx) -> None: if transform in self.transforms: self.transforms.remove(transform) - def parse(self): - """Parse `self.input` into a document tree.""" + def parse(self) -> None: + """Override the BaseReader parse method to call self.parser.parse_inline().""" self.document = document = self.new_document() self.parser.parse_inline(self.input, document) document.current_source = document.current_line = None diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 6c6d82b2910..09ee7e8ff28 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -66,7 +66,6 @@ def get_transforms(self) -> list[type[Transform]]: def parse_inline(self, inputstring: str, document: nodes.document) -> None: """Parse inline syntax from text and generate a document tree.""" - # Avoid "Literal block expected; none found." warnings. has_literal = inputstring.endswith('::') if has_literal: @@ -86,10 +85,9 @@ def parse_inline(self, inputstring: str, document: nodes.document) -> None: self.finish_parse() if has_literal: p = document[0] - assert(isinstance(p, nodes.paragraph)) + assert isinstance(p, nodes.paragraph) p += nodes.Text(':') - def parse(self, inputstring: str | StringList, document: nodes.document) -> None: """Parse text and generate a document tree.""" self.setup_parse(inputstring, document) # type: ignore[arg-type] diff --git a/sphinx/transforms/i18n.py b/sphinx/transforms/i18n.py index 70dda14cfff..1c0cf809b40 100644 --- a/sphinx/transforms/i18n.py +++ b/sphinx/transforms/i18n.py @@ -5,7 +5,6 @@ import contextlib from os import path from re import DOTALL, match -from textwrap import indent from typing import TYPE_CHECKING, Any, TypeVar from docutils import nodes From 0794348576353792681cc028720d0fec20e2f119 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Mon, 8 Apr 2024 22:39:26 +0200 Subject: [PATCH 08/27] Fix last ruff error --- sphinx/parsers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 09ee7e8ff28..01f038fc541 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -48,7 +48,8 @@ def set_application(self, app: Sphinx) -> None: def parse_inline(self, inputstring: str, document: nodes.document) -> None: """Parse the inline elements of a text block and generate a document tree.""" - raise NotImplementedError('Parser subclasses must implement parse_inline') + msg = 'Parser subclasses must implement parse_inline' + raise NotImplementedError(msg) class RSTParser(docutils.parsers.rst.Parser, Parser): From 358c6112188a260f62e4e3881c922de06f5b80e0 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Mon, 8 Apr 2024 22:48:56 +0200 Subject: [PATCH 09/27] Remove unused type:ignore annotation --- sphinx/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 01f038fc541..31f4ec8de5a 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -72,7 +72,7 @@ def parse_inline(self, inputstring: str, document: nodes.document) -> None: if has_literal: inputstring = inputstring[:-2] - self.setup_parse(inputstring, document) # type: ignore[arg-type] + self.setup_parse(inputstring, document) self.statemachine = states.RSTStateMachine( state_classes=self.state_classes, initial_state='Text', From 94b9b9d1eb43025e766fc4a610c7532d3f24674f Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 14 Apr 2024 14:00:04 +0200 Subject: [PATCH 10/27] Simplify parse_inline literal block handling for rST --- sphinx/parsers.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 31f4ec8de5a..4deb71d1722 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -68,9 +68,8 @@ def get_transforms(self) -> list[type[Transform]]: def parse_inline(self, inputstring: str, document: nodes.document) -> None: """Parse inline syntax from text and generate a document tree.""" # Avoid "Literal block expected; none found." warnings. - has_literal = inputstring.endswith('::') - if has_literal: - inputstring = inputstring[:-2] + if inputstring.endswith('::'): + inputstring = inputstring[:-1] self.setup_parse(inputstring, document) self.statemachine = states.RSTStateMachine( @@ -84,10 +83,6 @@ def parse_inline(self, inputstring: str, document: nodes.document) -> None: self.decorate(inputlines) self.statemachine.run(inputlines, document, inliner=self.inliner) self.finish_parse() - if has_literal: - p = document[0] - assert isinstance(p, nodes.paragraph) - p += nodes.Text(':') def parse(self, inputstring: str | StringList, document: nodes.document) -> None: """Parse text and generate a document tree.""" From 17455d643ffb17cdf85d542b481fab94a2a93be9 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Mon, 15 Apr 2024 23:13:47 +0200 Subject: [PATCH 11/27] Fix rendering for parsed-literals Fixes #12287 --- sphinx/transforms/i18n.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sphinx/transforms/i18n.py b/sphinx/transforms/i18n.py index 1c0cf809b40..80c6e050135 100644 --- a/sphinx/transforms/i18n.py +++ b/sphinx/transforms/i18n.py @@ -442,8 +442,10 @@ def apply(self, **kwargs: Any) -> None: node['uri'] = msgstr continue - # literalblock do not need to be parsed as they do not contain inline syntax - if isinstance(node, LITERAL_TYPE_NODES): + # literalblock do not need to be parsed as they do not contain inline syntax, + # except for parsed-literals, but they use the same node type, so we differentiate + # them based on their number of children. + if isinstance(node, LITERAL_TYPE_NODES) and len(node.children) <= 1: node.children = [nodes.Text(msgstr)] # for highlighting that expects .rawsource and .astext() are same. node.rawsource = node.astext() From 0fc3f28a0980640d9c20719a0ac71594e60b5f37 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 14 Apr 2024 16:17:46 +0200 Subject: [PATCH 12/27] Regression test for parsed literals translation See <https://github.com/sphinx-doc/sphinx/issues/12287> --- tests/roots/test-intl/literalblock.txt | 8 ++++++ .../test-intl/xx/LC_MESSAGES/literalblock.po | 10 +++++++ tests/test_builders/test_build_gettext.py | 2 ++ tests/test_intl/test_intl.py | 27 +++++++++++-------- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/tests/roots/test-intl/literalblock.txt b/tests/roots/test-intl/literalblock.txt index 583b5b61072..b930eb773e8 100644 --- a/tests/roots/test-intl/literalblock.txt +++ b/tests/roots/test-intl/literalblock.txt @@ -69,3 +69,11 @@ doctest blocks >>> if __name__ == '__main__': # if run this py file as python script ... main() # call main + +parsed literal +============== + +.. parsed-literal:: + + **this** *is* + `parsed literal`_ diff --git a/tests/roots/test-intl/xx/LC_MESSAGES/literalblock.po b/tests/roots/test-intl/xx/LC_MESSAGES/literalblock.po index d320d957e42..18f83185c7f 100644 --- a/tests/roots/test-intl/xx/LC_MESSAGES/literalblock.po +++ b/tests/roots/test-intl/xx/LC_MESSAGES/literalblock.po @@ -125,3 +125,13 @@ msgstr "" ">>> if __name__ == '__main__': # IF RUN THIS PY FILE AS PYTHON SCRIPT\n" "... main() # CALL MAIN" +msgid "parsed literal" +msgstr "PARSED LITERAL" + +msgid "" +"**this** *is*\n" +"`parsed literal`_" +msgstr "" +"**THIS** *IS*\n" +"`PARSED LITERAL`_" + diff --git a/tests/test_builders/test_build_gettext.py b/tests/test_builders/test_build_gettext.py index 8d759f01de3..15c9204134d 100644 --- a/tests/test_builders/test_build_gettext.py +++ b/tests/test_builders/test_build_gettext.py @@ -265,4 +265,6 @@ def test_gettext_literalblock_additional(app): "function\\n... sys.stdout.write('hello') # call write method of " "stdout object\\n>>>\\n>>> if __name__ == '__main__': # if run this py " 'file as python script\\n... main() # call main', + 'parsed literal', + '**this** *is*\\n`parsed literal`_', ] diff --git a/tests/test_intl/test_intl.py b/tests/test_intl/test_intl.py index ee4c9f752c5..bf501543213 100644 --- a/tests/test_intl/test_intl.py +++ b/tests/test_intl/test_intl.py @@ -241,7 +241,7 @@ def test_text_definition_terms(app): app.build() # --- definition terms: regression test for #975, #2198, #2205 result = (app.outdir / 'definition_terms.txt').read_text(encoding='utf8') - expect = ("13. I18N WITH DEFINITION TERMS" + expect = ("14. I18N WITH DEFINITION TERMS" "\n******************************\n" "\nSOME TERM" "\n THE CORRESPONDING DEFINITION\n" @@ -261,7 +261,7 @@ def test_text_glossary_term(app, warning): app.build() # --- glossary terms: regression test for #1090 result = (app.outdir / 'glossary_terms.txt').read_text(encoding='utf8') - expect = (r"""18. I18N WITH GLOSSARY TERMS + expect = (r"""19. I18N WITH GLOSSARY TERMS **************************** SOME NEW TERM @@ -296,7 +296,7 @@ def test_text_glossary_term_inconsistencies(app, warning): app.build() # --- glossary term inconsistencies: regression test for #1090 result = (app.outdir / 'glossary_terms_inconsistency.txt').read_text(encoding='utf8') - expect = ("19. I18N WITH GLOSSARY TERMS INCONSISTENCY" + expect = ("20. I18N WITH GLOSSARY TERMS INCONSISTENCY" "\n******************************************\n" "\n1. LINK TO *SOME NEW TERM*.\n") assert result == expect @@ -341,7 +341,7 @@ def test_text_seealso(app): app.build() # --- seealso result = (app.outdir / 'seealso.txt').read_text(encoding='utf8') - expect = ("12. I18N WITH SEEALSO" + expect = ("13. I18N WITH SEEALSO" "\n*********************\n" "\nSee also: SHORT TEXT 1\n" "\nSee also: LONG TEXT 1\n" @@ -358,13 +358,13 @@ def test_text_figure_captions(app): app.build() # --- figure captions: regression test for #940 result = (app.outdir / 'figure.txt').read_text(encoding='utf8') - expect = ("14. I18N WITH FIGURE CAPTION" + expect = ("15. I18N WITH FIGURE CAPTION" "\n****************************\n" "\n [image]MY CAPTION OF THE FIGURE\n" "\n MY DESCRIPTION PARAGRAPH1 OF THE FIGURE.\n" "\n MY DESCRIPTION PARAGRAPH2 OF THE FIGURE.\n" "\n" - "\n14.1. FIGURE IN THE BLOCK" + "\n15.1. FIGURE IN THE BLOCK" "\n=========================\n" "\nBLOCK\n" "\n [image]MY CAPTION OF THE FIGURE\n" @@ -372,7 +372,7 @@ def test_text_figure_captions(app): "\n MY DESCRIPTION PARAGRAPH2 OF THE FIGURE.\n" "\n" "\n" - "14.2. IMAGE URL AND ALT\n" + "15.2. IMAGE URL AND ALT\n" "=======================\n" "\n" "[image: I18N -> IMG][image]\n" @@ -380,11 +380,11 @@ def test_text_figure_captions(app): " [image: IMG -> I18N][image]\n" "\n" "\n" - "14.3. IMAGE ON SUBSTITUTION\n" + "15.3. IMAGE ON SUBSTITUTION\n" "===========================\n" "\n" "\n" - "14.4. IMAGE UNDER NOTE\n" + "15.4. IMAGE UNDER NOTE\n" "======================\n" "\n" "Note:\n" @@ -420,7 +420,7 @@ def test_text_docfields(app): app.build() # --- docfields result = (app.outdir / 'docfields.txt').read_text(encoding='utf8') - expect = ("21. I18N WITH DOCFIELDS" + expect = ("22. I18N WITH DOCFIELDS" "\n***********************\n" "\nclass Cls1\n" "\n Parameters:" @@ -1435,7 +1435,7 @@ def test_additional_targets_should_be_translated(app): """<span class="no">LIST</span>""") assert_count(expected_expr, result, 1) - # doctest block should not be translated but be highlighted + # doctest block should be translated and highlighted expected_expr = ( """<span class="gp">>>> </span>""" """<span class="kn">import</span> <span class="nn">sys</span> """ @@ -1445,6 +1445,11 @@ def test_additional_targets_should_be_translated(app): # '#noqa' should remain in literal blocks. assert_count("#noqa", result, 1) + # parsed literal should be translated + expected_expr = ('<strong>THIS</strong> <em>IS</em>\n' + '<a class="reference internal" href="#parsed-literal">PARSED LITERAL</a>') + assert_count(expected_expr, result, 1) + # [raw.txt] result = (app.outdir / 'raw.html').read_text(encoding='utf8') From fe7675f96d9a16b7dde669f4fcab1ff7949a041a Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 21 Apr 2024 22:27:20 +0200 Subject: [PATCH 13/27] Regression test for strange markup See <https://github.com/sphinx-doc/sphinx/pull/12277#issuecomment-2068190983> --- tests/roots/test-intl/markup.txt | 2 ++ tests/roots/test-intl/xx/LC_MESSAGES/markup.po | 3 +++ tests/test_intl/test_intl.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/tests/roots/test-intl/markup.txt b/tests/roots/test-intl/markup.txt index d167a042bca..6d6f6e51bbe 100644 --- a/tests/roots/test-intl/markup.txt +++ b/tests/roots/test-intl/markup.txt @@ -4,3 +4,5 @@ i18n with strange markup 1. title starting with 1. ------------------------- +A. Einstein was a really +smart dude. diff --git a/tests/roots/test-intl/xx/LC_MESSAGES/markup.po b/tests/roots/test-intl/xx/LC_MESSAGES/markup.po index ad6de9b4417..0235b458189 100644 --- a/tests/roots/test-intl/xx/LC_MESSAGES/markup.po +++ b/tests/roots/test-intl/xx/LC_MESSAGES/markup.po @@ -23,3 +23,6 @@ msgstr "I18N WITH STRANGE MARKUP" msgid "1. title starting with 1." msgstr "1. TITLE STARTING WITH 1." +msgid "A. Einstein was a really smart dude." +msgstr "A. EINSTEIN WAS A REALLY SMART DUDE." + diff --git a/tests/test_intl/test_intl.py b/tests/test_intl/test_intl.py index bf501543213..e58f5ab9382 100644 --- a/tests/test_intl/test_intl.py +++ b/tests/test_intl/test_intl.py @@ -1314,6 +1314,9 @@ def test_xml_strange_markup(app): title1, = subsec1.findall('title') assert_elem(title1, ['1. TITLE STARTING WITH 1.']) + pars = subsec1.findall('paragraph') + assert_elem(pars[0], ['A. EINSTEIN WAS A REALLY SMART DUDE.']) + @sphinx_intl @pytest.mark.sphinx('html') From 7f94dcf3b39591bb3f5be0fba2a77cae8a8c2943 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Wed, 8 May 2024 19:39:33 +0200 Subject: [PATCH 14/27] Directly use Inliner.parse() instead of 'Text' initial_state As recommended by @chrisjsewell --- sphinx/io.py | 3 +-- sphinx/parsers.py | 37 +++++++++++++++++++++++-------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/sphinx/io.py b/sphinx/io.py index 4fdcca09a04..135010b962b 100644 --- a/sphinx/io.py +++ b/sphinx/io.py @@ -146,8 +146,7 @@ def setup(self, app: Sphinx) -> None: def parse(self) -> None: """Override the BaseReader parse method to call self.parser.parse_inline().""" self.document = document = self.new_document() - self.parser.parse_inline(self.input, document) - document.current_source = document.current_line = None + self.parser.parse_inline(self.input, document, 1) class SphinxDummyWriter(UnfilteredWriter): diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 4deb71d1722..cfa7bb4f256 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -8,6 +8,7 @@ import docutils.parsers.rst from docutils import nodes from docutils.parsers.rst import states +from docutils.parsers.rst import languages from docutils.statemachine import StringList from docutils.transforms.universal import SmartQuotes @@ -46,7 +47,7 @@ def set_application(self, app: Sphinx) -> None: self.config = app.config self.env = app.env - def parse_inline(self, inputstring: str, document: nodes.document) -> None: + def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) -> None: """Parse the inline elements of a text block and generate a document tree.""" msg = 'Parser subclasses must implement parse_inline' raise NotImplementedError(msg) @@ -65,24 +66,32 @@ def get_transforms(self) -> list[type[Transform]]: transforms.remove(SmartQuotes) return transforms - def parse_inline(self, inputstring: str, document: nodes.document) -> None: + def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) -> None: """Parse inline syntax from text and generate a document tree.""" # Avoid "Literal block expected; none found." warnings. if inputstring.endswith('::'): inputstring = inputstring[:-1] - self.setup_parse(inputstring, document) - self.statemachine = states.RSTStateMachine( - state_classes=self.state_classes, - initial_state='Text', - debug=document.reporter.debug_flag, - ) - - inputlines = StringList([inputstring], document.current_source) - - self.decorate(inputlines) - self.statemachine.run(inputlines, document, inliner=self.inliner) - self.finish_parse() + language = languages.get_language( + document.settings.language_code, document.reporter) + if self.inliner is None: + inliner = states.Inliner() + else: + inliner = self.inliner + inliner.init_customizations(document.settings) + memo = states.Struct(document=document, + reporter=document.reporter, + language=language, + title_styles=[], + section_level=0, + section_bubble_up_kludge=False, + inliner=inliner) + memo.reporter.get_source_and_line = lambda x: (document.source, x) + textnodes, _ = inliner.parse(inputstring, lineno, memo, document) + p = nodes.paragraph(inputstring, '', *textnodes) + p.source = document.source + p.line = lineno + document.append(p) def parse(self, inputstring: str | StringList, document: nodes.document) -> None: """Parse text and generate a document tree.""" From 14e483357a35033af6fa6045d82632d2caccc5dd Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Wed, 8 May 2024 19:48:45 +0200 Subject: [PATCH 15/27] Add warning messages to doccument in parse_inline As recommended by @chrisjsewell. Because even if they are ignored by the i18n transform, parse_inline() could still be used in other places. --- sphinx/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index cfa7bb4f256..3741b23b8b2 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -87,11 +87,11 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) section_bubble_up_kludge=False, inliner=inliner) memo.reporter.get_source_and_line = lambda x: (document.source, x) - textnodes, _ = inliner.parse(inputstring, lineno, memo, document) + textnodes, messages = inliner.parse(inputstring, lineno, memo, document) p = nodes.paragraph(inputstring, '', *textnodes) p.source = document.source p.line = lineno - document.append(p) + document += [p] + messages def parse(self, inputstring: str | StringList, document: nodes.document) -> None: """Parse text and generate a document tree.""" From c81d310ea91870d2f8502e9bbf3ed9293c1cd2fa Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Thu, 9 May 2024 09:27:14 +0200 Subject: [PATCH 16/27] Fix ruff lints --- sphinx/parsers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 3741b23b8b2..802ce83a7da 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -7,8 +7,7 @@ import docutils.parsers import docutils.parsers.rst from docutils import nodes -from docutils.parsers.rst import states -from docutils.parsers.rst import languages +from docutils.parsers.rst import languages, states from docutils.statemachine import StringList from docutils.transforms.universal import SmartQuotes @@ -91,7 +90,7 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) p = nodes.paragraph(inputstring, '', *textnodes) p.source = document.source p.line = lineno - document += [p] + messages + document += [p, *messages] def parse(self, inputstring: str | StringList, document: nodes.document) -> None: """Parse text and generate a document tree.""" From 910dbd733315be5e15004444ac12b15f855a5d5c Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Thu, 9 May 2024 09:31:58 +0200 Subject: [PATCH 17/27] Apply ruff format patch --- sphinx/parsers.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 802ce83a7da..3ebc0a47248 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -71,20 +71,21 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) if inputstring.endswith('::'): inputstring = inputstring[:-1] - language = languages.get_language( - document.settings.language_code, document.reporter) + language = languages.get_language(document.settings.language_code, document.reporter) if self.inliner is None: inliner = states.Inliner() else: inliner = self.inliner inliner.init_customizations(document.settings) - memo = states.Struct(document=document, - reporter=document.reporter, - language=language, - title_styles=[], - section_level=0, - section_bubble_up_kludge=False, - inliner=inliner) + memo = states.Struct( + document=document, + reporter=document.reporter, + language=language, + title_styles=[], + section_level=0, + section_bubble_up_kludge=False, + inliner=inliner, + ) memo.reporter.get_source_and_line = lambda x: (document.source, x) textnodes, messages = inliner.parse(inputstring, lineno, memo, document) p = nodes.paragraph(inputstring, '', *textnodes) From 3b92bfb222a5279afbc23f23109e912db168f9eb Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Fri, 10 May 2024 18:39:56 +0200 Subject: [PATCH 18/27] Refactor to fix MyPy error --- sphinx/parsers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 3ebc0a47248..4dd362ab0db 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -71,6 +71,8 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) if inputstring.endswith('::'): inputstring = inputstring[:-1] + reporter = document.reporter + reporter.get_source_and_line = lambda x: (document.source, x) language = languages.get_language(document.settings.language_code, document.reporter) if self.inliner is None: inliner = states.Inliner() @@ -79,14 +81,13 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) inliner.init_customizations(document.settings) memo = states.Struct( document=document, - reporter=document.reporter, + reporter=reporter, language=language, title_styles=[], section_level=0, section_bubble_up_kludge=False, inliner=inliner, ) - memo.reporter.get_source_and_line = lambda x: (document.source, x) textnodes, messages = inliner.parse(inputstring, lineno, memo, document) p = nodes.paragraph(inputstring, '', *textnodes) p.source = document.source From 8c40fb64d55650f5898e192599d350ab019c77c7 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sat, 11 May 2024 17:35:16 +0200 Subject: [PATCH 19/27] Add tests for parse_inline --- tests/test_markup/test_parse_inline.py | 120 +++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tests/test_markup/test_parse_inline.py diff --git a/tests/test_markup/test_parse_inline.py b/tests/test_markup/test_parse_inline.py new file mode 100644 index 00000000000..fd76650ae90 --- /dev/null +++ b/tests/test_markup/test_parse_inline.py @@ -0,0 +1,120 @@ +import os +import pytest +from docutils.io import StringInput +from docutils import frontend, nodes + +from sphinx import addnodes, parsers +from sphinx.io import SphinxStandaloneReader +from sphinx.util.console import strip_colors + +@pytest.fixture +def document(app): + settings = frontend.get_default_settings(parsers.RSTParser) + settings.env = app.builder.env + reader = SphinxStandaloneReader() + reader.setup(app) + reader.source = StringInput(source_path="dummy/document.rst") + reader.settings = settings + document = reader.new_document() + return document + +@pytest.fixture +def parser(app): + parser = parsers.RSTParser() + parser.set_application(app) + return parser + +@pytest.mark.parametrize(('rst', 'expected'), [ + ( + # pep role + ':pep:`8`', + [addnodes.index, nodes.target, nodes.reference] + ), + ( + # rfc role + ':rfc:`2324`', + [addnodes.index, nodes.target, nodes.reference] + ), + ( + # correct interpretation of code with whitespace + '``code sample``', + [nodes.literal] + ), + ( + # no ampersands in guilabel + ':guilabel:`Foo`', + [nodes.inline] + ), + ( + # kbd role + ':kbd:`space`', + [nodes.literal] + ), + ( + # description list: simple + 'term\n description', + [nodes.Text] + ), + ( + # description list: with classifiers + 'term : class1 : class2\n description', + [nodes.Text] + ), + ( + # glossary (description list): multiple terms + '.. glossary::\n\n term1\n term2\n description', + [nodes.Text] + ), + ( + # basic inline markup + '**Strong** text and *emphasis*', + [nodes.strong, nodes.Text, nodes.emphasis] + ), + ( + # literal block + 'see this code block::\n\n hello world!', + [nodes.Text] + ), + ( + # missing literal block + 'see this code block::', + [nodes.Text] + ), + ( + # footnote reference + 'Reference a footnote [1]_', + [nodes.Text, nodes.footnote_reference] + ), +]) +def test_inline_no_error(rst, expected, parser, document): + parser.parse_inline(rst, document, 1) + assert len(document.children) == 1 + paragraph = document.children[0] + assert paragraph.__class__ == nodes.paragraph + assert len(paragraph.children) == len(expected) + for i, child in enumerate(paragraph.children): + assert child.__class__ == expected[i] + + +@pytest.mark.parametrize(('rst', 'expected'), [ + ( + # invalid unfinished literal + '``code sample', + 'WARNING: Inline literal start-string without end-string.' + ), +]) +def test_inline_errors(rst, expected, parser, document, warning): + lineno = 5 + expected = f'dummy/document.rst:{lineno}: {expected}' + parser.parse_inline(rst, document, lineno) + assert len(document.children) > 1 + paragraph = document.children[0] + messages = document.children[1:] + assert paragraph.__class__ == nodes.paragraph + for message in messages: + assert message.__class__ == nodes.system_message + warnings = getwarning(warning) + assert expected in warnings + +def getwarning(warnings): + return strip_colors(warnings.getvalue().replace(os.sep, '/')) From ca54a6c088e25fe4328be3b3ff1c9af1d979960e Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sat, 11 May 2024 17:56:09 +0200 Subject: [PATCH 20/27] Fix ruff lints --- tests/test_markup/test_parse_inline.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_markup/test_parse_inline.py b/tests/test_markup/test_parse_inline.py index fd76650ae90..b2b81169058 100644 --- a/tests/test_markup/test_parse_inline.py +++ b/tests/test_markup/test_parse_inline.py @@ -1,29 +1,33 @@ import os + import pytest -from docutils.io import StringInput from docutils import frontend, nodes +from docutils.io import StringInput from sphinx import addnodes, parsers from sphinx.io import SphinxStandaloneReader from sphinx.util.console import strip_colors -@pytest.fixture + +@pytest.fixture() def document(app): settings = frontend.get_default_settings(parsers.RSTParser) settings.env = app.builder.env reader = SphinxStandaloneReader() reader.setup(app) - reader.source = StringInput(source_path="dummy/document.rst") + reader.source = StringInput(source_path='dummy/document.rst') reader.settings = settings document = reader.new_document() return document -@pytest.fixture + +@pytest.fixture() def parser(app): parser = parsers.RSTParser() parser.set_application(app) return parser + @pytest.mark.parametrize(('rst', 'expected'), [ ( # pep role @@ -116,5 +120,6 @@ def test_inline_errors(rst, expected, parser, document, warning): warnings = getwarning(warning) assert expected in warnings + def getwarning(warnings): return strip_colors(warnings.getvalue().replace(os.sep, '/')) From 7de9a40bf65e4cc8d41c3433756ed4408c618bf6 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sat, 11 May 2024 18:25:46 +0200 Subject: [PATCH 21/27] Skip parse_inline() tests with docutils < 0.19 As frontend.get_defaut_settings() function recommended in docutils's docs [1] is not there. [1] https://docutils.sourceforge.io/docs/dev/hacking.html#parsing-the-document --- tests/test_markup/test_parse_inline.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_markup/test_parse_inline.py b/tests/test_markup/test_parse_inline.py index b2b81169058..1c0dde145b9 100644 --- a/tests/test_markup/test_parse_inline.py +++ b/tests/test_markup/test_parse_inline.py @@ -1,5 +1,6 @@ import os +import docutils import pytest from docutils import frontend, nodes from docutils.io import StringInput @@ -8,6 +9,10 @@ from sphinx.io import SphinxStandaloneReader from sphinx.util.console import strip_colors +docutils_version = pytest.mark.skipif( + docutils.__version_info__ < (0, 19), reason="at least docutils 0.19 required" +) + @pytest.fixture() def document(app): @@ -28,6 +33,7 @@ def parser(app): return parser +@docutils_version() @pytest.mark.parametrize(('rst', 'expected'), [ ( # pep role @@ -100,6 +106,7 @@ def test_inline_no_error(rst, expected, parser, document): assert child.__class__ == expected[i] +@docutils_version() @pytest.mark.parametrize(('rst', 'expected'), [ ( # invalid unfinished literal From 7500a0094373cb02e3bfa94e563cdd1936e87e60 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sat, 11 May 2024 18:27:51 +0200 Subject: [PATCH 22/27] Try to ignore incorrect mypy lint --- sphinx/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 4dd362ab0db..b76831efbe1 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -72,7 +72,7 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) inputstring = inputstring[:-1] reporter = document.reporter - reporter.get_source_and_line = lambda x: (document.source, x) + reporter.get_source_and_line = lambda x: (document.source, x) # type: ignore[attr-defined] language = languages.get_language(document.settings.language_code, document.reporter) if self.inliner is None: inliner = states.Inliner() From e7e7fadde10395c1a9cad79c2e6833b660a5b57f Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 12 May 2024 12:18:51 +0200 Subject: [PATCH 23/27] Remove unneeded inliner, title and section options of memo --- sphinx/parsers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index b76831efbe1..309afb0de90 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -83,10 +83,6 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) document=document, reporter=reporter, language=language, - title_styles=[], - section_level=0, - section_bubble_up_kludge=False, - inliner=inliner, ) textnodes, messages = inliner.parse(inputstring, lineno, memo, document) p = nodes.paragraph(inputstring, '', *textnodes) From dba713d67b9a6545059de8b03e3b4efac98049f3 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 12 May 2024 12:37:36 +0200 Subject: [PATCH 24/27] Add test for substitution reference for parse_inline --- tests/test_markup/test_parse_inline.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_markup/test_parse_inline.py b/tests/test_markup/test_parse_inline.py index 1c0dde145b9..8e9ece00694 100644 --- a/tests/test_markup/test_parse_inline.py +++ b/tests/test_markup/test_parse_inline.py @@ -95,6 +95,11 @@ def parser(app): 'Reference a footnote [1]_', [nodes.Text, nodes.footnote_reference] ), + ( + # substitution reference + 'here is a |substituted| text', + [nodes.Text, nodes.substitution_reference, nodes.Text] + ), ]) def test_inline_no_error(rst, expected, parser, document): parser.parse_inline(rst, document, 1) From 2c37780c06b1191fdf9eef1f052cd757956e69ce Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 19 May 2024 10:43:54 +0200 Subject: [PATCH 25/27] Simplify parse_inline for RST --- sphinx/parsers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 309afb0de90..0ed43da59b1 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -73,7 +73,7 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) reporter = document.reporter reporter.get_source_and_line = lambda x: (document.source, x) # type: ignore[attr-defined] - language = languages.get_language(document.settings.language_code, document.reporter) + language = languages.get_language(document.settings.language_code, reporter) if self.inliner is None: inliner = states.Inliner() else: @@ -86,8 +86,7 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) ) textnodes, messages = inliner.parse(inputstring, lineno, memo, document) p = nodes.paragraph(inputstring, '', *textnodes) - p.source = document.source - p.line = lineno + p.source, p.line = document.source, lineno document += [p, *messages] def parse(self, inputstring: str | StringList, document: nodes.document) -> None: From 3d5441dc2743c216e2f520b95c4f072cddfa83da Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 19 May 2024 10:44:10 +0200 Subject: [PATCH 26/27] Add test for titles in parse_inline --- tests/test_markup/test_parse_inline.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_markup/test_parse_inline.py b/tests/test_markup/test_parse_inline.py index 8e9ece00694..3f1c86b3ddc 100644 --- a/tests/test_markup/test_parse_inline.py +++ b/tests/test_markup/test_parse_inline.py @@ -90,6 +90,11 @@ def parser(app): 'see this code block::', [nodes.Text] ), + ( + # section title + 'This is a title\n================\n', + [nodes.Text] + ), ( # footnote reference 'Reference a footnote [1]_', From d84d4e65dbdf56132ada5f9c96a8c5c64f69228e Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <nicolas@club1.fr> Date: Sun, 19 May 2024 21:23:12 +0200 Subject: [PATCH 27/27] Fix <unknown> source file in translated warnings --- sphinx/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 0ed43da59b1..7375059d68c 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -72,7 +72,7 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) inputstring = inputstring[:-1] reporter = document.reporter - reporter.get_source_and_line = lambda x: (document.source, x) # type: ignore[attr-defined] + reporter.get_source_and_line = lambda x: (document['source'], x) # type: ignore[attr-defined] language = languages.get_language(document.settings.language_code, reporter) if self.inliner is None: inliner = states.Inliner() @@ -86,7 +86,7 @@ def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) ) textnodes, messages = inliner.parse(inputstring, lineno, memo, document) p = nodes.paragraph(inputstring, '', *textnodes) - p.source, p.line = document.source, lineno + p.source, p.line = document['source'], lineno document += [p, *messages] def parse(self, inputstring: str | StringList, document: nodes.document) -> None: