From 7508de119b1b65d84f0c6fa0cc7c3d08d368734d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pinard?= Date: Thu, 8 Sep 2022 23:21:33 +0200 Subject: [PATCH 1/5] extend __all__ members to template rendering Fixes #10809 * the option autosummary_ignore_module_all when set to False adds members to module's members entry that will be used for autodoc, but otherwise it ignores it. As such, if a class is available in the __all__, it won't be generated. * This commit aims at extending the __all__ handling not only to members, but also to corresponding attribute types (function, classes, exceptions, modules) * In short, the imported_members option is set to True if the object has __all__ member and autosummary is set to have autosummary_ignore_module_all set to False --- sphinx/ext/autosummary/generate.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 9b9abdf26d1..8069dc2cbed 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -321,17 +321,26 @@ def get_modules(obj: Any) -> tuple[list[str], list[str]]: if doc.objtype == 'module': scanner = ModuleScanner(app, obj) ns['members'] = scanner.scan(imported_members) + + respect_module_all = not app.config.autosummary_ignore_module_all + imported_members = imported_members or '__all__' in dir(obj) and respect_module_all + ns['functions'], ns['all_functions'] = \ get_members(obj, {'function'}, imported=imported_members) ns['classes'], ns['all_classes'] = \ get_members(obj, {'class'}, imported=imported_members) ns['exceptions'], ns['all_exceptions'] = \ get_members(obj, {'exception'}, imported=imported_members) + if respect_module_all: + ns['modules'], ns['all_modules'] = \ + get_members(obj, {'module'}, imported=imported_members) ns['attributes'], ns['all_attributes'] = \ get_module_attrs(ns['members']) ispackage = hasattr(obj, '__path__') if ispackage and recursive: - ns['modules'], ns['all_modules'] = get_modules(obj) + modules, all_modules = get_modules(obj) + ns['modules'] = list(set(modules + ns["modules"])) + ns['all_modules'] = list(set(all_modules + ns["all_modules"])) elif doc.objtype == 'class': ns['members'] = dir(obj) ns['inherited_members'] = \ From b82b5944de10b26d24e08eda83fa2c25166017b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pinard?= Date: Thu, 8 Sep 2022 23:34:47 +0200 Subject: [PATCH 2/5] Bugfix: create empty module and all_modules when autosummary_ignore_module_all is True --- sphinx/ext/autosummary/generate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 8069dc2cbed..db0c7288632 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -334,6 +334,8 @@ def get_modules(obj: Any) -> tuple[list[str], list[str]]: if respect_module_all: ns['modules'], ns['all_modules'] = \ get_members(obj, {'module'}, imported=imported_members) + else: + ns['modules'], ns['all_modules'] = [], [] ns['attributes'], ns['all_attributes'] = \ get_module_attrs(ns['members']) ispackage = hasattr(obj, '__path__') From a35fdfd65c6cce6ec0245c4ce343218401a2541b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pinard?= Date: Wed, 5 Apr 2023 17:32:13 +0200 Subject: [PATCH 3/5] Add test and and entry to CHANGES --- CHANGES | 3 +++ sphinx/ext/autosummary/generate.py | 15 +++++------- .../index.rst | 4 ++-- .../autosummary_dummy_package_all/__init__.py | 13 ++++++++++ .../autosummary_dummy_module.py | 20 ++++++++++++++++ .../extra_dummy_module.py | 20 ++++++++++++++++ .../test-ext-autosummary-module_all/conf.py | 8 +++++++ .../test-ext-autosummary-module_all/index.rst | 8 +++++++ tests/test_ext_autosummary.py | 24 +++++++++++++++++++ 9 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py create mode 100644 tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/autosummary_dummy_module.py create mode 100644 tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/extra_dummy_module.py create mode 100644 tests/roots/test-ext-autosummary-module_all/conf.py create mode 100644 tests/roots/test-ext-autosummary-module_all/index.rst diff --git a/CHANGES b/CHANGES index d90ba2f0dad..39314e7a68e 100644 --- a/CHANGES +++ b/CHANGES @@ -27,6 +27,9 @@ Deprecated Features added -------------- +* #10811: Autosummary: extend ``__all__`` to imported members for template rendering + when option ``autosummary_ignore_module_all`` is set to ``False``. Patch by + Clement Pinard * #11147: Add a ``content_offset`` parameter to ``nested_parse_with_titles()``, allowing for correct line numbers during nested parsing. Patch by Jeremy Maitin-Shepard diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index db0c7288632..47543051783 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -323,7 +323,7 @@ def get_modules(obj: Any) -> tuple[list[str], list[str]]: ns['members'] = scanner.scan(imported_members) respect_module_all = not app.config.autosummary_ignore_module_all - imported_members = imported_members or '__all__' in dir(obj) and respect_module_all + imported_members = imported_members or ('__all__' in dir(obj) and respect_module_all) ns['functions'], ns['all_functions'] = \ get_members(obj, {'function'}, imported=imported_members) @@ -331,18 +331,15 @@ def get_modules(obj: Any) -> tuple[list[str], list[str]]: get_members(obj, {'class'}, imported=imported_members) ns['exceptions'], ns['all_exceptions'] = \ get_members(obj, {'exception'}, imported=imported_members) - if respect_module_all: - ns['modules'], ns['all_modules'] = \ - get_members(obj, {'module'}, imported=imported_members) - else: - ns['modules'], ns['all_modules'] = [], [] ns['attributes'], ns['all_attributes'] = \ get_module_attrs(ns['members']) ispackage = hasattr(obj, '__path__') if ispackage and recursive: - modules, all_modules = get_modules(obj) - ns['modules'] = list(set(modules + ns["modules"])) - ns['all_modules'] = list(set(all_modules + ns["all_modules"])) + if imported_members and respect_module_all: + ns['modules'], ns['all_modules'] = \ + get_members(obj, {'module'}, imported=imported_members) + else: + ns['modules'], ns['all_modules'] = get_modules(obj) elif doc.objtype == 'class': ns['members'] = dir(obj) ns['inherited_members'] = \ diff --git a/tests/roots/test-ext-autosummary-imported_members/index.rst b/tests/roots/test-ext-autosummary-imported_members/index.rst index 608ca2954f7..1c551265ead 100644 --- a/tests/roots/test-ext-autosummary-imported_members/index.rst +++ b/tests/roots/test-ext-autosummary-imported_members/index.rst @@ -1,5 +1,5 @@ -test-ext-autosummary-mock_imports -================================= +test-ext-autosummary-imported_members +===================================== .. autosummary:: :toctree: generated diff --git a/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py new file mode 100644 index 00000000000..ea6a9005462 --- /dev/null +++ b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py @@ -0,0 +1,13 @@ +from .autosummary_dummy_module import Bar, foo, public_foo, PublicBar +from . import extra_dummy_module + +def baz(): + """Baz function""" + pass + + +def public_baz(): + """Public Baz function""" + + +__all__ = ["PublicBar", "public_foo", "public_baz", "extra_dummy_module"] diff --git a/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/autosummary_dummy_module.py b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/autosummary_dummy_module.py new file mode 100644 index 00000000000..ef89e22f455 --- /dev/null +++ b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/autosummary_dummy_module.py @@ -0,0 +1,20 @@ +class Bar: + """Bar class""" + + pass + + +class PublicBar: + """Public Bar class""" + + pass + + +def foo(): + """Foo function""" + pass + + +def public_foo(): + """Public Foo function""" + pass diff --git a/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/extra_dummy_module.py b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/extra_dummy_module.py new file mode 100644 index 00000000000..ef89e22f455 --- /dev/null +++ b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/extra_dummy_module.py @@ -0,0 +1,20 @@ +class Bar: + """Bar class""" + + pass + + +class PublicBar: + """Public Bar class""" + + pass + + +def foo(): + """Foo function""" + pass + + +def public_foo(): + """Public Foo function""" + pass diff --git a/tests/roots/test-ext-autosummary-module_all/conf.py b/tests/roots/test-ext-autosummary-module_all/conf.py new file mode 100644 index 00000000000..c6ff53419b9 --- /dev/null +++ b/tests/roots/test-ext-autosummary-module_all/conf.py @@ -0,0 +1,8 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) + +extensions = ['sphinx.ext.autosummary'] +autosummary_generate = True +autosummary_ignore_module_all = False diff --git a/tests/roots/test-ext-autosummary-module_all/index.rst b/tests/roots/test-ext-autosummary-module_all/index.rst new file mode 100644 index 00000000000..cd638ad3549 --- /dev/null +++ b/tests/roots/test-ext-autosummary-module_all/index.rst @@ -0,0 +1,8 @@ +test-ext-autosummary-module_all +=============================== + +.. autosummary:: + :toctree: generated + :recursive: + + autosummary_dummy_package_all diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index f8c2de495ce..389c7fa2151 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -545,6 +545,30 @@ def test_autosummary_imported_members(app, status, warning): sys.modules.pop('autosummary_dummy_package', None) +@pytest.mark.sphinx('dummy', testroot='ext-autosummary-module_all') +def test_autosummary_module_all(app, status, warning): + try: + app.build() + # generated/foo is generated successfully + assert app.env.get_doctree('generated/autosummary_dummy_package_all') + module = (app.srcdir / 'generated' / 'autosummary_dummy_package_all.rst').read_text(encoding='utf8') + assert (' .. autosummary::\n' + ' \n' + ' PublicBar\n' + ' \n' in module) + assert (' .. autosummary::\n' + ' \n' + ' public_foo\n' + ' public_baz\n' + ' \n' in module) + assert ('.. autosummary::\n' + ' :toctree:\n' + ' :recursive:\n\n' + ' extra_dummy_module\n\n' in module) + finally: + sys.modules.pop('autosummary_dummy_package_all', None) + + @pytest.mark.sphinx(testroot='ext-autodoc', confoverrides={'extensions': ['sphinx.ext.autosummary']}) def test_generate_autosummary_docs_property(app): From dd618b9fc7f9e3162c110597046afa403461bb25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pinard?= Date: Wed, 5 Apr 2023 20:01:47 +0200 Subject: [PATCH 4/5] Improve autosummary recursive module * When ignore_module_all is False, search for imported modules (possibly renamed) and in discovered modules, public modules are then the one mentioned in __all__ * In a more general usecase, skip all modules that are overwritten in the package namespace --- sphinx/ext/autosummary/generate.py | 51 ++++++++++++++++--- .../autosummary_dummy_package_all/__init__.py | 1 - tests/test_ext_autosummary.py | 2 +- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 47543051783..1daecee6b5e 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -300,9 +300,18 @@ def get_module_attrs(members: Any) -> tuple[list[str], list[str]]: pass # give up if ModuleAnalyzer fails to parse code return public, attrs - def get_modules(obj: Any) -> tuple[list[str], list[str]]: + def get_modules( + obj: Any, + skip: Sequence[str], + public_members: Sequence[str] | None = None, + ) -> tuple[list[str], list[str]]: items: list[str] = [] + public: list[str] = [] for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__): + + if modname in skip: + # module was overwritten in __init__.py, so not accessible + continue fullname = name + '.' + modname try: module = import_module(fullname) @@ -312,7 +321,12 @@ def get_modules(obj: Any) -> tuple[list[str], list[str]]: pass items.append(fullname) - public = [x for x in items if not x.split('.')[-1].startswith('_')] + if public_members is not None: + if modname in public_members: + public.append(fullname) + else: + if not modname.startswith('_'): + public.append(fullname) return public, items ns: dict[str, Any] = {} @@ -335,11 +349,36 @@ def get_modules(obj: Any) -> tuple[list[str], list[str]]: get_module_attrs(ns['members']) ispackage = hasattr(obj, '__path__') if ispackage and recursive: - if imported_members and respect_module_all: - ns['modules'], ns['all_modules'] = \ - get_members(obj, {'module'}, imported=imported_members) + # Use members that are not modules as skip list, because it would then mean + # that module was overwritten in the package namespace + skip = ( + ns["all_functions"] + + ns["all_classes"] + + ns["all_exceptions"] + + ns["all_attributes"] + ) + + # If respect_module_all and module has a __all__ attribute, first get + # modules that were explicitly imported. Next, find the rest with the + # get_modules method, but only put in "public" modules that are in the + # __all__ list + # + # Otherwise, use get_modules method normally + if respect_module_all and '__all__' in dir(obj): + imported_modules, all_imported_modules = \ + get_members(obj, {'module'}, imported=True) + skip += all_imported_modules + imported_modules = [name + '.' + modname for modname in imported_modules] + all_imported_modules = \ + [name + '.' + modname for modname in all_imported_modules] + public_members = getall(obj) else: - ns['modules'], ns['all_modules'] = get_modules(obj) + imported_modules, all_imported_modules = [], [] + public_members = None + + modules, all_modules = get_modules(obj, skip=skip, public_members=public_members) + ns['modules'] = imported_modules + modules + ns["all_modules"] = all_imported_modules + all_modules elif doc.objtype == 'class': ns['members'] = dir(obj) ns['inherited_members'] = \ diff --git a/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py index ea6a9005462..bd307a649da 100644 --- a/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py +++ b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py @@ -1,5 +1,4 @@ from .autosummary_dummy_module import Bar, foo, public_foo, PublicBar -from . import extra_dummy_module def baz(): """Baz function""" diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 389c7fa2151..69b4b76bcfc 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -564,7 +564,7 @@ def test_autosummary_module_all(app, status, warning): assert ('.. autosummary::\n' ' :toctree:\n' ' :recursive:\n\n' - ' extra_dummy_module\n\n' in module) + ' autosummary_dummy_package_all.extra_dummy_module\n\n' in module) finally: sys.modules.pop('autosummary_dummy_package_all', None) From e9fad74c3e9251a9e2684c466c3e9c4a6736b171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pinard?= Date: Wed, 5 Apr 2023 20:13:51 +0200 Subject: [PATCH 5/5] Fix linting --- sphinx/ext/autosummary/generate.py | 3 +-- .../autosummary_dummy_package_all/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 1daecee6b5e..b74ce014294 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -303,8 +303,7 @@ def get_module_attrs(members: Any) -> tuple[list[str], list[str]]: def get_modules( obj: Any, skip: Sequence[str], - public_members: Sequence[str] | None = None, - ) -> tuple[list[str], list[str]]: + public_members: Sequence[str] | None = None) -> tuple[list[str], list[str]]: items: list[str] = [] public: list[str] = [] for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__): diff --git a/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py index bd307a649da..82f2060fb58 100644 --- a/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py +++ b/tests/roots/test-ext-autosummary-module_all/autosummary_dummy_package_all/__init__.py @@ -1,4 +1,5 @@ -from .autosummary_dummy_module import Bar, foo, public_foo, PublicBar +from .autosummary_dummy_module import Bar, PublicBar, foo, public_foo + def baz(): """Baz function"""