From d89b859379a5771cc849c8da06a0df7570b9cbeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 3 Aug 2023 10:54:32 +0200 Subject: [PATCH 01/14] Fix ``sphinx.ext.autodoc.preserve_defaults`` extension. --- sphinx/ext/autodoc/preserve_defaults.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index 0d58ae54e00..1964ba40d29 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -72,7 +72,18 @@ def update_defvalue(app: Sphinx, obj: Any, bound_method: bool) -> None: try: function = get_function_def(obj) - assert function is not None # for mypy + if function is None: + # If the object is a built-in, we won't be always able to recover + # the function definition. This may happen for instance if *obj* + # is a dataclass, in which case *function* is None. + return + + # Although get_function_def() is expected to return ast.FunctionDef, + # it sometimes returns ast.ClassDef objects which do not have an + # "args" field. In the future, get_function_def() should be accurately + # modified so that it always return an ast.FunctionDef object or None, + # but for now, we will catch AttributeError instead and silently ignore + # them. if function.args.defaults or function.args.kw_defaults: sig = inspect.signature(obj) defaults = list(function.args.defaults) From 835254a8c9f8a44f9b5cdedb54985fa340497a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 3 Aug 2023 13:32:37 +0200 Subject: [PATCH 02/14] Fix ``sphinx.ext.autodoc.preserve_defaults``. - Add support for ``async def`` functions. - Add support for ``lambda`` functions. - Rename ``get_function_def`` to ``get_arguments`` and update its signature accordingly. Previously, ``get_function_def`` incorrectly extracted ``ast.FunctionDef`` from an arbitrary object. - Silently suppress syntax errors arising from multi-line parsing. --- sphinx/ext/autodoc/preserve_defaults.py | 61 +++++++++++++------ .../target/preserve_defaults.py | 28 +++++++++ tests/test_ext_autodoc_preserve_defaults.py | 42 +++++++++++++ 3 files changed, 113 insertions(+), 18 deletions(-) diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index 1964ba40d29..295c7f9468e 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -8,6 +8,7 @@ import ast import inspect +from types import LambdaType from typing import Any import sphinx @@ -27,8 +28,15 @@ def __repr__(self) -> str: return self.name -def get_function_def(obj: Any) -> ast.FunctionDef | None: - """Get FunctionDef object from living object. +_LAMBDA_NAME = (lambda: None).__name__ + + +def _islambda(v): + return isinstance(v, LambdaType) and v.__name__ == _LAMBDA_NAME + + +def get_arguments(obj: Any) -> ast.arguments | None: + """Get ast.arguments object from living object. This tries to parse original code for living object and returns AST node for given *obj*. """ @@ -38,12 +46,35 @@ def get_function_def(obj: Any) -> ast.FunctionDef | None: # subject is placed inside class or block. To read its docstring, # this adds if-block before the declaration. module = ast.parse('if True:\n' + source) - return module.body[0].body[0] # type: ignore + subject = module.body[0].body[0] else: module = ast.parse(source) - return module.body[0] # type: ignore + subject = module.body[0] except (OSError, TypeError): # failed to load source code return None + except SyntaxError: + if _islambda(obj): + # most likely a multi-line arising from detecting a lambda, e.g.: + # + # class Foo: + # x = property( + # lambda self: 1, doc="...")) + return None + + # Other syntax errors that are not due to the fact that we are + # documenting a lambda function are propagated (in particular, + # if a lambda function is renamed by the user, the SyntaxError is + # propagated). + raise + + def _get_arguments(x: ast.AST) -> ast.arguments | None: + if isinstance(x, (ast.AsyncFunctionDef, ast.FunctionDef, ast.Lambda)): + return x.args + if isinstance(x, (ast.Assign, ast.AnnAssign)): + return _get_arguments(x.value) + return None + + return _get_arguments(subject) def get_default_value(lines: list[str], position: ast.AST) -> str | None: @@ -66,28 +97,22 @@ def update_defvalue(app: Sphinx, obj: Any, bound_method: bool) -> None: try: lines = inspect.getsource(obj).splitlines() if lines[0].startswith((' ', r'\t')): - lines.insert(0, '') # insert a dummy line to follow what get_function_def() does. + lines.insert(0, '') # insert a dummy line to follow what get_arguments() does. except (OSError, TypeError): lines = [] try: - function = get_function_def(obj) - if function is None: + args = get_arguments(obj) + if args is None: # If the object is a built-in, we won't be always able to recover - # the function definition. This may happen for instance if *obj* - # is a dataclass, in which case *function* is None. + # the function definition and its arguments. This happens if *obj* + # is the `__init__` method generated automatically for dataclasses. return - # Although get_function_def() is expected to return ast.FunctionDef, - # it sometimes returns ast.ClassDef objects which do not have an - # "args" field. In the future, get_function_def() should be accurately - # modified so that it always return an ast.FunctionDef object or None, - # but for now, we will catch AttributeError instead and silently ignore - # them. - if function.args.defaults or function.args.kw_defaults: + if args.defaults or args.kw_defaults: sig = inspect.signature(obj) - defaults = list(function.args.defaults) - kw_defaults = list(function.args.kw_defaults) + defaults = list(args.defaults) + kw_defaults = list(args.kw_defaults) parameters = list(sig.parameters.values()) for i, param in enumerate(parameters): if param.default is param.empty: diff --git a/tests/roots/test-ext-autodoc/target/preserve_defaults.py b/tests/roots/test-ext-autodoc/target/preserve_defaults.py index 2759f27e89a..86e103840d2 100644 --- a/tests/roots/test-ext-autodoc/target/preserve_defaults.py +++ b/tests/roots/test-ext-autodoc/target/preserve_defaults.py @@ -30,3 +30,31 @@ def clsmeth(cls, name: str = CONSTANT, sentinel: Any = SENTINEL, now: datetime = datetime.now(), color: int = 0xFFFFFF, *, kwarg1, kwarg2 = 0xFFFFFF) -> None: """docstring""" + + +get_sentinel = lambda custom=SENTINEL: custom +"""docstring""" + + +class MultiLine: + """docstring""" + + # The properties will raise a silent SyntaxError because "lambda self: 1" + # will be detected as a function to update the default values of. However, + # only prop3 will not fail because it's on a single line whereas the others + # will fail to parse. + + prop1 = property( + lambda self: 1, doc="docstring") + + prop2 = property( + lambda self: 2, doc="docstring" + ) + + prop3 = property(lambda self: 3, doc="docstring") + + prop4 = (property + (lambda self: 4, doc="docstring")) + + prop5 = property\ + (lambda self: 5, doc="docstring") diff --git a/tests/test_ext_autodoc_preserve_defaults.py b/tests/test_ext_autodoc_preserve_defaults.py index d4cef7a6ea6..659b21c0925 100644 --- a/tests/test_ext_autodoc_preserve_defaults.py +++ b/tests/test_ext_autodoc_preserve_defaults.py @@ -40,6 +40,42 @@ def test_preserve_defaults(app): ' docstring', '', '', + '.. py:class:: MultiLine()', + ' :module: target.preserve_defaults', + '', + ' docstring', + '', + '', + ' .. py:property:: MultiLine.prop1', + ' :module: target.preserve_defaults', + '', + ' docstring', + '', + '', + ' .. py:property:: MultiLine.prop2', + ' :module: target.preserve_defaults', + '', + ' docstring', + '', + '', + ' .. py:property:: MultiLine.prop3', + ' :module: target.preserve_defaults', + '', + ' docstring', + '', + '', + ' .. py:property:: MultiLine.prop4', + ' :module: target.preserve_defaults', + '', + ' docstring', + '', + '', + ' .. py:property:: MultiLine.prop5', + ' :module: target.preserve_defaults', + '', + ' docstring', + '', + '', '.. py:function:: foo(name: str = CONSTANT, sentinel: ~typing.Any = SENTINEL, ' 'now: ~datetime.datetime = datetime.now(), color: int = %s, *, kwarg1, ' 'kwarg2=%s) -> None' % (color, color), @@ -47,4 +83,10 @@ def test_preserve_defaults(app): '', ' docstring', '', + '', + '.. py:function:: get_sentinel(custom=SENTINEL)', + ' :module: target.preserve_defaults', + '', + ' docstring', + '', ] From 36d65931024f90d8e7cca42c8d6c90ec5776d37d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 3 Aug 2023 13:46:18 +0200 Subject: [PATCH 03/14] update CHANGES --- CHANGES | 12 ++++++++++++ sphinx/ext/autodoc/preserve_defaults.py | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 645a6cd4c9d..03ed7e55728 100644 --- a/CHANGES +++ b/CHANGES @@ -9,6 +9,12 @@ Dependencies Incompatible changes -------------------- +* #11459: ``sphinx.ext.autodoc.preserve_defaults.get_function_def`` is + renamed ``sphinx.ext.autodoc.preserve_defaults.get_arguments`` and its + signature is updated accordingly. It now extracts the ``ast.arguments`` from + a function or a lambda function object. + Patch by Bénédikt Tran. + Deprecated ---------- @@ -36,12 +42,18 @@ Features added Patch by Rouslan Korneychuk. * 10938: doctest: Add :confval:`doctest_show_successes` option. Patch by Trey Hunner. +* #11459: ``sphinx.ext.autodoc.preserve_defaults``: Add support for + ``async def`` functions and lambda functions. + Patch by Bénédikt Tran. Bugs fixed ---------- * #11077: graphviz: Fix relative links from within the graph. Patch by Ralf Grubenmann. +* #11459: ``sphinx.ext.autodoc.preserve_defaults``: fix lambda functions + used inside ``@property`` should be ignored in a multi-line context. + Patch by Bénédikt Tran. * #11529: Line Block in LaTeX builder outputs spurious empty token. Patch by Adrian Vollmer. diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index 295c7f9468e..3622aa9fbc5 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -46,7 +46,7 @@ def get_arguments(obj: Any) -> ast.arguments | None: # subject is placed inside class or block. To read its docstring, # this adds if-block before the declaration. module = ast.parse('if True:\n' + source) - subject = module.body[0].body[0] + subject = module.body[0].body[0] # type: ignore[attr-defined] else: module = ast.parse(source) subject = module.body[0] @@ -67,7 +67,7 @@ def get_arguments(obj: Any) -> ast.arguments | None: # propagated). raise - def _get_arguments(x: ast.AST) -> ast.arguments | None: + def _get_arguments(x: Any) -> ast.arguments | None: if isinstance(x, (ast.AsyncFunctionDef, ast.FunctionDef, ast.Lambda)): return x.args if isinstance(x, (ast.Assign, ast.AnnAssign)): From dfac4ab144141cf1d54ae7bb6b00973aaebca822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 4 Aug 2023 08:20:19 +0200 Subject: [PATCH 04/14] update comment --- sphinx/ext/autodoc/preserve_defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index 3622aa9fbc5..6f71fb683a3 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -57,8 +57,8 @@ def get_arguments(obj: Any) -> ast.arguments | None: # most likely a multi-line arising from detecting a lambda, e.g.: # # class Foo: - # x = property( - # lambda self: 1, doc="...")) + # x = property( + # lambda self: 1, doc="...") return None # Other syntax errors that are not due to the fact that we are From ee79aaafbd1364c0a6428327d1d2de4381b111e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:42:21 +0200 Subject: [PATCH 05/14] fix whitespace --- sphinx/ext/autodoc/preserve_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index a0e216eb3a4..7af9de26a6e 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from typing import Any - + from sphinx.application import Sphinx logger = logging.getLogger(__name__) From 1506dcf1d313796a86d1525e90e6f7b8e7ec2e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:45:23 +0200 Subject: [PATCH 06/14] update CHANGES --- CHANGES | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 5c7f8607308..e68592d68d4 100644 --- a/CHANGES +++ b/CHANGES @@ -10,11 +10,6 @@ Dependencies Incompatible changes -------------------- -* #11459: ``sphinx.ext.autodoc.preserve_defaults.get_function_def`` is - renamed ``sphinx.ext.autodoc.preserve_defaults.get_arguments`` and its - signature is updated accordingly. It now extracts the ``ast.arguments`` from - a function or a lambda function object. - Patch by Bénédikt Tran. Deprecated ---------- @@ -33,6 +28,12 @@ Deprecated ``sphinx.builders.html.StandaloneHTMLBuilder.script_files``. Use ``sphinx.application.Sphinx.add_css_file()`` and ``sphinx.application.Sphinx.add_js_file()`` instead. +* #11459: ``sphinx.ext.autodoc.preserve_defaults.get_function_def`` is + deprecated and should not be used to get ``ast.FunctionDef`` objects + and their ``ast.arguments`` objects. + Use ``sphinx.ext.autodoc.preserve_defaults.get_arguments`` to extract + ``ast.arguments`` from a function or a lambda function object. + Patch by Bénédikt Tran. Features added -------------- From 4d5404ec6938f0eb456916a77556e7a347148fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:57:47 +0200 Subject: [PATCH 07/14] add deprecation warnings --- sphinx/ext/autodoc/preserve_defaults.py | 29 ++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index 7af9de26a6e..90c224e21ef 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -8,10 +8,12 @@ import ast import inspect +import warnings from types import LambdaType from typing import TYPE_CHECKING import sphinx +from sphinx.deprecation import RemovedInSphinx90Warning from sphinx.locale import __ from sphinx.pycode.ast import unparse as ast_unparse from sphinx.util import logging @@ -19,7 +21,7 @@ if TYPE_CHECKING: from typing import Any - from sphinx.application import Sphinx + from sphinx.application import Sphinx logger = logging.getLogger(__name__) @@ -32,6 +34,31 @@ def __repr__(self) -> str: return self.name +def get_function_def(obj: Any) -> ast.FunctionDef | None: + """Get FunctionDef object from living object. + + This tries to parse original code for living object and returns + AST node for given *obj*. + """ + warnings.warn(f'{__name__}.get_function_def is deprecated and incorrect.' + f' Use sphinx.ext.autodoc.get_arguments to get extract AST' + f' arguments objects from a lambda or a regular function.', + RemovedInSphinx90Warning, stacklevel=2) + + try: + source = inspect.getsource(obj) + if source.startswith((' ', r'\t')): + # subject is placed inside class or block. To read its docstring, + # this adds if-block before the declaration. + module = ast.parse('if True:\n' + source) + return module.body[0].body[0] # type: ignore[attr-defined] + else: + module = ast.parse(source) + return module.body[0] # type: ignore[return-value] + except (OSError, TypeError): # failed to load source code + return None + + _LAMBDA_NAME = (lambda: None).__name__ From 71ca5d4e019a2f02ffe0b46828b7628f17c2d7f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:03:02 +0200 Subject: [PATCH 08/14] add deprecation warnings --- sphinx/ext/autodoc/preserve_defaults.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index 90c224e21ef..e7d8f3f51db 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -40,9 +40,11 @@ def get_function_def(obj: Any) -> ast.FunctionDef | None: This tries to parse original code for living object and returns AST node for given *obj*. """ - warnings.warn(f'{__name__}.get_function_def is deprecated and incorrect.' - f' Use sphinx.ext.autodoc.get_arguments to get extract AST' - f' arguments objects from a lambda or a regular function.', + warnings.warn('sphinx.ext.autodoc.preserve_defaults.get_function_def is' + ' incorrect and scheduled for removal.' + ' Use sphinx.ext.autodoc.preserve_defaults.get_arguments to' + ' extract AST arguments objects from a lambda or regular' + ' function.', RemovedInSphinx90Warning, stacklevel=2) try: From 4d66b7849294473d0f714e87200eed606e93bd84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:04:14 +0200 Subject: [PATCH 09/14] fix doc --- CHANGES | 1 - sphinx/ext/autodoc/preserve_defaults.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index e68592d68d4..774ec40d68b 100644 --- a/CHANGES +++ b/CHANGES @@ -10,7 +10,6 @@ Dependencies Incompatible changes -------------------- - Deprecated ---------- diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index e7d8f3f51db..f9c156d6f5e 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -44,8 +44,7 @@ def get_function_def(obj: Any) -> ast.FunctionDef | None: ' incorrect and scheduled for removal.' ' Use sphinx.ext.autodoc.preserve_defaults.get_arguments to' ' extract AST arguments objects from a lambda or regular' - ' function.', - RemovedInSphinx90Warning, stacklevel=2) + ' function.', RemovedInSphinx90Warning, stacklevel=2) try: source = inspect.getsource(obj) From 18da1e5bf95ff1025ab847d2478e674ee9cc4d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:34:24 +0200 Subject: [PATCH 10/14] update tests --- .../preserve_defaults_special_constructs.py | 51 +++++++++ tests/test_ext_autodoc_preserve_defaults.py | 100 ++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py diff --git a/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py b/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py new file mode 100644 index 00000000000..d342967e7e6 --- /dev/null +++ b/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from collections import namedtuple +from dataclasses import field, dataclass +from typing import TypedDict, NamedTuple + + +#: docstring +SENTINEL = object() + + +#: docstring +ze_lambda = lambda z=SENTINEL: None + + +def foo(x, y, z=SENTINEL): + """docstring""" + + +@dataclass +class DataClass: + """docstring""" + a: int + b: object = SENTINEL + c: list[int] = field(default_factory=lambda: [1, 2, 3]) + + +@dataclass(init=False) +class DataClassNoInit: + """docstring""" + a: int + b: object = SENTINEL + c: list[int] = field(default_factory=lambda: [1, 2, 3]) + + +class MyTypedDict(TypedDict): + """docstring""" + a: int + b: object + c: list[int] + + +class MyNamedTuple1(NamedTuple): + """docstring""" + a: int + b: object = object() + c: list[int] = [1, 2, 3] + + +class MyNamedTuple2(namedtuple('Base', ('a', 'b'), defaults=(0, SENTINEL))): + """docstring""" diff --git a/tests/test_ext_autodoc_preserve_defaults.py b/tests/test_ext_autodoc_preserve_defaults.py index 659b21c0925..70b61462871 100644 --- a/tests/test_ext_autodoc_preserve_defaults.py +++ b/tests/test_ext_autodoc_preserve_defaults.py @@ -90,3 +90,103 @@ def test_preserve_defaults(app): ' docstring', '', ] + + +@pytest.mark.sphinx('html', testroot='ext-autodoc', + confoverrides={'autodoc_preserve_defaults': True}) +def test_preserve_defaults_special_constructs(app): + options = {"members": None} + actual = do_autodoc(app, 'module', 'target.preserve_defaults_special_constructs', options) + + # * dataclasses.dataclass: + # - __init__ source code is not available + # - default values specified at class level are not discovered + # - values wrapped in a field(...) expression cannot be analyzed + # easily even if annotations were to be parsed + # * typing.NamedTuple: + # - __init__ source code is not available + # - default values specified at class level are not discovered + # * collections.namedtuple: + # - default values are specified as "default=(d1, d2, ...)" + # + # In the future, it might be possible to find some additional default + # values by parsing the source code of the annotations but the task is + # rather complex. + + assert list(actual) == [ + '', + '.. py:module:: target.preserve_defaults_special_constructs', + '', + '', + '.. py:class:: DataClass(' + 'a: int, b: object = , c: list[int] = )', + ' :module: target.preserve_defaults_special_constructs', + '', + ' docstring', + '', + '', + '.. py:class:: DataClassNoInit()', + ' :module: target.preserve_defaults_special_constructs', + '', + ' docstring', + '', + '', + '.. py:class:: MyNamedTuple1(' + 'a: int, b: object = , c: list[int] = [1, 2, 3])', + ' :module: target.preserve_defaults_special_constructs', + '', + ' docstring', + '', + '', + ' .. py:attribute:: MyNamedTuple1.a', + ' :module: target.preserve_defaults_special_constructs', + ' :type: int', + '', + ' Alias for field number 0', + '', + '', + ' .. py:attribute:: MyNamedTuple1.b', + ' :module: target.preserve_defaults_special_constructs', + ' :type: object', + '', + ' Alias for field number 1', + '', + '', + ' .. py:attribute:: MyNamedTuple1.c', + ' :module: target.preserve_defaults_special_constructs', + ' :type: list[int]', + '', + ' Alias for field number 2', + '', + '', + '.. py:class:: MyNamedTuple2(a=0, b=)', + ' :module: target.preserve_defaults_special_constructs', + '', + ' docstring', + '', + '', + '.. py:class:: MyTypedDict', + ' :module: target.preserve_defaults_special_constructs', + '', + ' docstring', + '', + '', + '.. py:data:: SENTINEL', + ' :module: target.preserve_defaults_special_constructs', + ' :value: ', + '', + ' docstring', + '', + '', + '.. py:function:: foo(x, y, z=SENTINEL)', + ' :module: target.preserve_defaults_special_constructs', + '', + ' docstring', + '', + '', + '.. py:function:: ze_lambda(z=SENTINEL)', + ' :module: target.preserve_defaults_special_constructs', + '', + ' docstring', + '', + ] From d90dfe4e0b6b68bd394e1cfa774bfd357e487276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:37:16 +0200 Subject: [PATCH 11/14] fix lint --- .../target/preserve_defaults_special_constructs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py b/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py index d342967e7e6..00858ddeac6 100644 --- a/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py +++ b/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py @@ -1,8 +1,8 @@ from __future__ import annotations from collections import namedtuple -from dataclasses import field, dataclass -from typing import TypedDict, NamedTuple +from dataclasses import dataclass, field +from typing import NamedTuple, TypedDict #: docstring From 6bf30113e88e7eb64890e7244a553dad94de8a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:42:32 +0200 Subject: [PATCH 12/14] fix lint --- .../target/preserve_defaults_special_constructs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py b/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py index 00858ddeac6..0fdb11ac874 100644 --- a/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py +++ b/tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py @@ -4,7 +4,6 @@ from dataclasses import dataclass, field from typing import NamedTuple, TypedDict - #: docstring SENTINEL = object() From 9cefefa599a126792b7eacf5cd7b69333c304b1e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Aug 2023 23:55:38 +0100 Subject: [PATCH 13/14] Implementation updates --- CHANGES | 15 ++---- doc/extdev/deprecated.rst | 5 ++ sphinx/ext/autodoc/preserve_defaults.py | 68 ++++++++++++------------- 3 files changed, 43 insertions(+), 45 deletions(-) diff --git a/CHANGES b/CHANGES index d00bcfbaf85..1b738a00b1b 100644 --- a/CHANGES +++ b/CHANGES @@ -27,11 +27,7 @@ Deprecated ``sphinx.builders.html.StandaloneHTMLBuilder.script_files``. Use ``sphinx.application.Sphinx.add_css_file()`` and ``sphinx.application.Sphinx.add_js_file()`` instead. -* #11459: ``sphinx.ext.autodoc.preserve_defaults.get_function_def`` is - deprecated and should not be used to get ``ast.FunctionDef`` objects - and their ``ast.arguments`` objects. - Use ``sphinx.ext.autodoc.preserve_defaults.get_arguments`` to extract - ``ast.arguments`` from a function or a lambda function object. +* #11459: Deprecate ``sphinx.ext.autodoc.preserve_defaults.get_function_def()``. Patch by Bénédikt Tran. Features added @@ -65,18 +61,12 @@ Features added details. Patch by Bénédikt Tran. * Allow running Sphinx with ``python -m sphinx build ...``. -* #11459: ``sphinx.ext.autodoc.preserve_defaults``: Add support for - ``async def`` functions and lambda functions. - Patch by Bénédikt Tran. Bugs fixed ---------- * #11077: graphviz: Fix relative links from within the graph. Patch by Ralf Grubenmann. -* #11459: ``sphinx.ext.autodoc.preserve_defaults``: fix lambda functions - used inside ``@property`` should be ignored in a multi-line context. - Patch by Bénédikt Tran. * #11529: Line Block in LaTeX builder outputs spurious empty token. Patch by Adrian Vollmer. * #11196: autosummary: Summary line extraction failed with "e.g." @@ -100,6 +90,9 @@ Bugs fixed Patch by Bénédikt Tran. * #11591: Fix support for C coverage in ``sphinx.ext.coverage`` extension. Patch by Stephen Finucane. +* #11459: Fix support for async and lambda functions in + ``sphinx.ext.autodoc.preserve_defaults``. + Patch by Bénédikt Tran. Testing ------- diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst index 08477ab5eaf..f90389cc4b2 100644 --- a/doc/extdev/deprecated.rst +++ b/doc/extdev/deprecated.rst @@ -22,6 +22,11 @@ The following is a list of deprecated interfaces. - Removed - Alternatives + * - ``sphinx.ext.autodoc.preserve_defaults.get_function_def()`` + - 7.2 + - 9.0 + - N/A (replacement is private) + * - ``sphinx.builders.html.StandaloneHTMLBuilder.css_files`` - 7.2 - 9.0 diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index f9c156d6f5e..9a766a6c97f 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -8,8 +8,8 @@ import ast import inspect +import types import warnings -from types import LambdaType from typing import TYPE_CHECKING import sphinx @@ -24,6 +24,7 @@ from sphinx.application import Sphinx logger = logging.getLogger(__name__) +_LAMBDA_NAME = (lambda: None).__name__ class DefaultValue: @@ -41,14 +42,14 @@ def get_function_def(obj: Any) -> ast.FunctionDef | None: AST node for given *obj*. """ warnings.warn('sphinx.ext.autodoc.preserve_defaults.get_function_def is' - ' incorrect and scheduled for removal.' - ' Use sphinx.ext.autodoc.preserve_defaults.get_arguments to' + ' deprecated and scheduled for removal in Sphinx 9.' + ' Use sphinx.ext.autodoc.preserve_defaults._get_arguments() to' ' extract AST arguments objects from a lambda or regular' ' function.', RemovedInSphinx90Warning, stacklevel=2) try: source = inspect.getsource(obj) - if source.startswith((' ', r'\t')): + if source.startswith((' ', '\t')): # subject is placed inside class or block. To read its docstring, # this adds if-block before the declaration. module = ast.parse('if True:\n' + source) @@ -60,53 +61,51 @@ def get_function_def(obj: Any) -> ast.FunctionDef | None: return None -_LAMBDA_NAME = (lambda: None).__name__ - - -def _islambda(v): - return isinstance(v, LambdaType) and v.__name__ == _LAMBDA_NAME +def _get_arguments(obj: Any, /) -> ast.arguments | None: + """Parse 'ast.arguments' from an object. - -def get_arguments(obj: Any) -> ast.arguments | None: - """Get ast.arguments object from living object. - This tries to parse original code for living object and returns - AST node for given *obj*. + This tries to parse the original code for an object and returns + an 'ast.arguments' node. """ try: source = inspect.getsource(obj) - if source.startswith((' ', r'\t')): - # subject is placed inside class or block. To read its docstring, - # this adds if-block before the declaration. + if source.startswith((' ', '\t')): + # 'obj' is in some indented block. module = ast.parse('if True:\n' + source) subject = module.body[0].body[0] # type: ignore[attr-defined] else: module = ast.parse(source) subject = module.body[0] - except (OSError, TypeError): # failed to load source code + except (OSError, TypeError): + # bail; failed to load source for 'obj'. return None except SyntaxError: - if _islambda(obj): - # most likely a multi-line arising from detecting a lambda, e.g.: + if _is_lambda(obj): + # Most likely a multi-line arising from detecting a lambda, e.g.: # - # class Foo: + # class Egg: # x = property( # lambda self: 1, doc="...") return None # Other syntax errors that are not due to the fact that we are - # documenting a lambda function are propagated (in particular, - # if a lambda function is renamed by the user, the SyntaxError is - # propagated). + # documenting a lambda function are propagated + # (in particular if a lambda is renamed by the user). raise - def _get_arguments(x: Any) -> ast.arguments | None: - if isinstance(x, (ast.AsyncFunctionDef, ast.FunctionDef, ast.Lambda)): - return x.args - if isinstance(x, (ast.Assign, ast.AnnAssign)): - return _get_arguments(x.value) - return None + return _get_arguments_inner(subject) + + +def _is_lambda(x, /): + return isinstance(x, types.LambdaType) and x.__name__ == _LAMBDA_NAME + - return _get_arguments(subject) +def _get_arguments_inner(x: Any, /) -> ast.arguments | None: + if isinstance(x, (ast.AsyncFunctionDef, ast.FunctionDef, ast.Lambda)): + return x.args + if isinstance(x, (ast.Assign, ast.AnnAssign)): + return _get_arguments_inner(x.value) + return None def get_default_value(lines: list[str], position: ast.AST) -> str | None: @@ -128,13 +127,14 @@ def update_defvalue(app: Sphinx, obj: Any, bound_method: bool) -> None: try: lines = inspect.getsource(obj).splitlines() - if lines[0].startswith((' ', r'\t')): - lines.insert(0, '') # insert a dummy line to follow what get_arguments() does. + if lines[0].startswith((' ', '\t')): + # insert a dummy line to follow what _get_arguments() does. + lines.insert(0, '') except (OSError, TypeError): lines = [] try: - args = get_arguments(obj) + args = _get_arguments(obj) if args is None: # If the object is a built-in, we won't be always able to recover # the function definition and its arguments. This happens if *obj* From 88f57e1f12c10cacb53b97fb17aac360fd2d74dd Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 17 Aug 2023 01:53:19 +0100 Subject: [PATCH 14/14] whitespace --- sphinx/ext/autodoc/preserve_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py index 9a766a6c97f..e46644277ec 100644 --- a/sphinx/ext/autodoc/preserve_defaults.py +++ b/sphinx/ext/autodoc/preserve_defaults.py @@ -129,7 +129,7 @@ def update_defvalue(app: Sphinx, obj: Any, bound_method: bool) -> None: lines = inspect.getsource(obj).splitlines() if lines[0].startswith((' ', '\t')): # insert a dummy line to follow what _get_arguments() does. - lines.insert(0, '') + lines.insert(0, '') except (OSError, TypeError): lines = []