From 519cc078fdedd848e798a6ee3176c59b0f7da97e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 13 Jun 2021 15:46:22 +0900 Subject: [PATCH 1/3] Revert the removal of sphinx.util:force_decode() After the release of 4.0.0, some 3rd party extensions have became not working with the latest Sphinx because `force_decode()` function was removed. It was deprecated since Sphinx-2.0 and warned for the removal since 3.0. This reverts the removal and extends its deprecation period to 5.0.0. I hope it helps users of these extensions. --- CHANGES | 3 +++ doc/extdev/deprecated.rst | 2 +- sphinx/util/__init__.py | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 5b890fe8420..3739db03572 100644 --- a/CHANGES +++ b/CHANGES @@ -13,6 +13,9 @@ Deprecated Features added -------------- +* Revert the removal of ``sphinx.util:force_decode()`` to become some 3rd party + extensions available again during 5.0 + Bugs fixed ---------- diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst index 9e17b9fb4f0..e37c2b2ba4b 100644 --- a/doc/extdev/deprecated.rst +++ b/doc/extdev/deprecated.rst @@ -1015,7 +1015,7 @@ The following is a list of deprecated interfaces. * - ``sphinx.util.force_decode()`` - 2.0 - - 4.0 + - 5.0 - N/A * - ``sphinx.util.get_matching_docs()`` diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index 7e24909ee83..170c554d77b 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -337,6 +337,23 @@ def parselinenos(spec: str, total: int) -> List[int]: return items +def force_decode(string: str, encoding: str) -> str: + """Forcibly get a unicode string out of a bytestring.""" + warnings.warn('force_decode() is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) + if isinstance(string, bytes): + try: + if encoding: + string = string.decode(encoding) + else: + # try decoding with utf-8, should only work for real UTF-8 + string = string.decode() + except UnicodeError: + # last resort -- can't fail + string = string.decode('latin1') + return string + + def rpartition(s: str, t: str) -> Tuple[str, str]: """Similar to str.rpartition from 2.5, but doesn't return the separator.""" warnings.warn('rpartition() is now deprecated.', RemovedInSphinx50Warning, stacklevel=2) From 2de7c35573f557798fc65c17c8830178321c0a78 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 13 Jun 2021 23:11:47 +0900 Subject: [PATCH 2/3] Fix #9322: KeyError is raised on PropagateDescDomain transform PropageteDescDomain applies the domain name from the "domain" attribute of parent node (desc node) to the desc_signature node. The structure has longly generated by ObjectDescription. But it must not be a new rule. This allows to build document that contains non standard doctree. --- CHANGES | 1 + sphinx/transforms/post_transforms/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 6710a63a5a3..73808f7c0fc 100644 --- a/CHANGES +++ b/CHANGES @@ -17,6 +17,7 @@ Bugs fixed ---------- * #9313: LaTeX: complex table with merged cells broken since 4.0 +* #9322: KeyError is raised on PropagateDescDomain transform Testing -------- diff --git a/sphinx/transforms/post_transforms/__init__.py b/sphinx/transforms/post_transforms/__init__.py index dde06fae1be..7f4b3e3a691 100644 --- a/sphinx/transforms/post_transforms/__init__.py +++ b/sphinx/transforms/post_transforms/__init__.py @@ -249,7 +249,8 @@ class PropagateDescDomain(SphinxPostTransform): def run(self, **kwargs: Any) -> None: for node in self.document.traverse(addnodes.desc_signature): - node['classes'].append(node.parent['domain']) + if node.parent.get('domain'): + node['classes'].append(node.parent['domain']) def setup(app: Sphinx) -> Dict[str, Any]: From 50f5280603fa9c66253f13e53dd7e1b4a0263447 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Wed, 16 Jun 2021 22:08:30 +0200 Subject: [PATCH 3/3] C, keyword changes - Add new keywords from C23. - Add c_extra_keywords as confval. - Move macro names from keywords to c_extra_keywords. Fixes sphinx-doc/sphinx#9354 --- CHANGES | 6 ++++++ doc/usage/configuration.rst | 8 ++++++++ sphinx/domains/c.py | 26 ++++++++++++++++++++++---- tests/test_domain_c.py | 25 ++++++++++++++++++------- 4 files changed, 54 insertions(+), 11 deletions(-) diff --git a/CHANGES b/CHANGES index 9a7f854e34a..62e1f60ee2f 100644 --- a/CHANGES +++ b/CHANGES @@ -13,6 +13,10 @@ Deprecated Features added -------------- +* C, add C23 keywords ``_Decimal32``, ``_Decimal64``, and ``_Decimal128``. +* #9354: C, add :confval:`c_extra_keywords` to allow user-defined keywords + during parsing. + Bugs fixed ---------- @@ -21,6 +25,8 @@ Bugs fixed * #9313: LaTeX: complex table with merged cells broken since 4.0 * #9305: LaTeX: backslash may cause Improper discretionary list pdf build error with Japanese engines +* #9354: C, remove special macro names from the keyword list. + See also :confval:`c_extra_keywords`. Testing -------- diff --git a/doc/usage/configuration.rst b/doc/usage/configuration.rst index a233e0fda9b..3c58da30878 100644 --- a/doc/usage/configuration.rst +++ b/doc/usage/configuration.rst @@ -2670,6 +2670,14 @@ Options for the C domain .. versionadded:: 3.0 +.. confval:: c_extra_keywords + + A list of identifiers to be recognized as keywords by the C parser. + It defaults to ``['alignas', 'alignof', 'bool', 'complex', 'imaginary', + 'noreturn', 'static_assert', 'thread_local']``. + + .. versionadded:: 4.0.3 + .. confval:: c_allow_pre_v3 A boolean (default ``False``) controlling whether to parse and try to diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index b0711f68e98..6ebfe04c4f0 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -54,10 +54,15 @@ 'else', 'enum', 'extern', 'float', 'for', 'goto', 'if', 'inline', 'int', 'long', 'register', 'restrict', 'return', 'short', 'signed', 'sizeof', 'static', 'struct', 'switch', 'typedef', 'union', 'unsigned', 'void', 'volatile', 'while', - '_Alignas', 'alignas', '_Alignof', 'alignof', '_Atomic', '_Bool', 'bool', - '_Complex', 'complex', '_Generic', '_Imaginary', 'imaginary', - '_Noreturn', 'noreturn', '_Static_assert', 'static_assert', - '_Thread_local', 'thread_local', + '_Alignas', '_Alignof', '_Atomic', '_Bool', '_Complex', + '_Decimal32', '_Decimal64', '_Decimal128', + '_Generic', '_Imaginary', '_Noreturn', '_Static_assert', '_Thread_local', +] +# These are only keyword'y when the corresponding headers are included. +# They are used as default value for c_extra_keywords. +_macroKeywords = [ + 'alignas', 'alignof', 'bool', 'complex', 'imaginary', 'noreturn', 'static_assert', + 'thread_local', ] # these are ordered by preceedence @@ -2535,6 +2540,12 @@ def _parse_nested_name(self) -> ASTNestedName: if identifier in _keywords: self.fail("Expected identifier in nested name, " "got keyword: %s" % identifier) + if self.matched_text in self.config.c_extra_keywords: + msg = "Expected identifier, got user-defined keyword: %s." \ + + " Remove it from c_extra_keywords to allow it as identifier.\n" \ + + "Currently c_extra_keywords is %s." + self.fail(msg % (self.matched_text, + str(self.config.c_extra_keywords))) ident = ASTIdentifier(identifier) names.append(ident) @@ -2711,6 +2722,12 @@ def _parse_declarator_name_suffix( if self.matched_text in _keywords: self.fail("Expected identifier, " "got keyword: %s" % self.matched_text) + if self.matched_text in self.config.c_extra_keywords: + msg = "Expected identifier, got user-defined keyword: %s." \ + + " Remove it from c_extra_keywords to allow it as identifier.\n" \ + + "Currently c_extra_keywords is %s." + self.fail(msg % (self.matched_text, + str(self.config.c_extra_keywords))) identifier = ASTIdentifier(self.matched_text) declId = ASTNestedName([identifier], rooted=False) else: @@ -3877,6 +3894,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_domain(CDomain) app.add_config_value("c_id_attributes", [], 'env') app.add_config_value("c_paren_attributes", [], 'env') + app.add_config_value("c_extra_keywords", _macroKeywords, 'env') app.add_post_transform(AliasTransform) app.add_config_value("c_allow_pre_v3", False, 'env') diff --git a/tests/test_domain_c.py b/tests/test_domain_c.py index ef4858786de..a891e18e29e 100644 --- a/tests/test_domain_c.py +++ b/tests/test_domain_c.py @@ -15,16 +15,20 @@ from sphinx import addnodes from sphinx.addnodes import desc -from sphinx.domains.c import DefinitionError, DefinitionParser, Symbol, _id_prefix, _max_id +from sphinx.domains.c import (DefinitionError, DefinitionParser, Symbol, _id_prefix, + _macroKeywords, _max_id) from sphinx.ext.intersphinx import load_mappings, normalize_intersphinx_mapping from sphinx.testing import restructuredtext from sphinx.testing.util import assert_node +class Config: + c_id_attributes = ["id_attr", 'LIGHTGBM_C_EXPORT'] + c_paren_attributes = ["paren_attr"] + c_extra_keywords = _macroKeywords + + def parse(name, string): - class Config: - c_id_attributes = ["id_attr", 'LIGHTGBM_C_EXPORT'] - c_paren_attributes = ["paren_attr"] parser = DefinitionParser(string, location=None, config=Config()) parser.allowFallbackExpressionParsing = False ast = parser.parse_declaration(name, name) @@ -114,9 +118,6 @@ def check(name, input, idDict, output=None, key=None, asTextOutput=None): def test_expressions(): def exprCheck(expr, output=None): - class Config: - c_id_attributes = ["id_attr"] - c_paren_attributes = ["paren_attr"] parser = DefinitionParser(expr, location=None, config=Config()) parser.allowFallbackExpressionParsing = False ast = parser.parse_expression() @@ -522,6 +523,16 @@ def test_attributes(): check('function', 'LIGHTGBM_C_EXPORT int LGBM_BoosterFree(int handle)', {1: 'LGBM_BoosterFree'}) + +def test_extra_keywords(): + with pytest.raises(DefinitionError, + match='Expected identifier, got user-defined keyword: complex.'): + parse('function', 'void f(int complex)') + with pytest.raises(DefinitionError, + match='Expected identifier, got user-defined keyword: complex.'): + parse('function', 'void complex(void)') + + # def test_print(): # # used for getting all the ids out for checking # for a in ids: