From c727fccd9bad37abdd04b29c58f3cf8d49848362 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Thu, 7 Sep 2023 14:53:11 -0400 Subject: [PATCH 1/6] Leverage `importlib.reload` for reloading modules Rather than directly manipulating `sys.modules` ourselves, rely on `importlib.reload` to do it, as it works in more cases (particularly for Rust extension modules built with PyO3). Also, import modules with `typing.TYPE_CHECKING` set to `False` first, and reload them with it set to `True`. This way, if the reload fails, things are likely to still be in a valid state from the first, successful import. --- sphinx/ext/autodoc/importer.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index 0bbde6b57d5..b8ccad72e27 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import importlib import sys import traceback @@ -84,22 +85,19 @@ def import_object(modname: str, objpath: list[str], objtype: str = '', while module is None: try: orig_modules = frozenset(sys.modules) + import_module(modname, warningiserror=warningiserror) + + # Try reloading modules with ``typing.TYPE_CHECKING == True``. try: - # try importing with ``typing.TYPE_CHECKING == True`` typing.TYPE_CHECKING = True - module = import_module(modname, warningiserror=warningiserror) - except ImportError: - # if that fails (e.g. circular import), retry with - # ``typing.TYPE_CHECKING == False`` after reverting - # changes made to ``sys.modules`` by the failed try - for m in [m for m in sys.modules if m not in orig_modules]: - sys.modules.pop(m) - - typing.TYPE_CHECKING = False - module = import_module(modname, warningiserror=warningiserror) + # Ignore failures; we've already successfully loaded these modules. + with contextlib.suppress(Exception): + new_modules = frozenset(sys.modules) - orig_modules + for m in new_modules: + importlib.reload(sys.modules[m]) finally: - # ensure ``typing.TYPE_CHECKING == False`` typing.TYPE_CHECKING = False + module = sys.modules[modname] logger.debug('[autodoc] import %s => %r', modname, module) except ImportError as exc: logger.debug('[autodoc] import %s => failed', modname) From e909d52080b0dcd553d96945a8a3a7b03e9f0036 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 13 Sep 2023 02:39:44 +0100 Subject: [PATCH 2/6] Add escape hatch --- sphinx/ext/autodoc/importer.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index b8ccad72e27..781a1950a50 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -4,6 +4,7 @@ import contextlib import importlib +import os import sys import traceback import typing @@ -22,6 +23,8 @@ ) if TYPE_CHECKING: + from types import ModuleType + from sphinx.ext.autodoc import ObjectMember logger = logging.getLogger(__name__) @@ -70,6 +73,19 @@ def import_module(modname: str, warningiserror: bool = False) -> Any: raise ImportError(exc, traceback.format_exc()) from exc +def _reload_module(module: ModuleType, warningiserror: bool = False) -> Any: + """ + Call importlib.reload(module), convert exceptions to ImportError + """ + try: + with logging.skip_warningiserror(not warningiserror): + return importlib.reload(module) + except BaseException as exc: + # Importing modules may cause any side effects, including + # SystemExit, so we need to catch all errors. + raise ImportError(exc, traceback.format_exc()) from exc + + def import_object(modname: str, objpath: list[str], objtype: str = '', attrgetter: Callable[[Any, str], Any] = safe_getattr, warningiserror: bool = False) -> Any: @@ -84,17 +100,19 @@ def import_object(modname: str, objpath: list[str], objtype: str = '', objpath = list(objpath) while module is None: try: - orig_modules = frozenset(sys.modules) + original_module_names = frozenset(sys.modules) import_module(modname, warningiserror=warningiserror) + new_modules = [m for m in sys.modules if m not in original_module_names] + if os.environ.get('SPHINX_AUTODOC_DO_NOT_RELOAD_MODULES'): + new_modules = [] # Try reloading modules with ``typing.TYPE_CHECKING == True``. try: typing.TYPE_CHECKING = True - # Ignore failures; we've already successfully loaded these modules. - with contextlib.suppress(Exception): - new_modules = frozenset(sys.modules) - orig_modules + # Ignore failures; we've already successfully loaded these modules + with contextlib.suppress(ImportError): for m in new_modules: - importlib.reload(sys.modules[m]) + _reload_module(sys.modules[m]) finally: typing.TYPE_CHECKING = False module = sys.modules[modname] From 159ac90cb05258e620c4e22c5eb97ff8fb86801e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 13 Sep 2023 02:58:01 +0100 Subject: [PATCH 3/6] Cython --- tests/test_ext_autodoc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_ext_autodoc.py b/tests/test_ext_autodoc.py index 02869b8bee2..770ae416d0a 100644 --- a/tests/test_ext_autodoc.py +++ b/tests/test_ext_autodoc.py @@ -2103,9 +2103,9 @@ def test_singledispatchmethod_automethod(app): ] -@pytest.mark.skipif(sys.version_info[:2] >= (3, 11), - reason=('cython does not support python-3.11 yet. ' - 'see https://github.com/cython/cython/issues/4365')) +@pytest.mark.skipif(sys.version_info[:2] >= (3, 12), + reason=('Cython does not support Python 3.12 yet. ' + 'See https://github.com/cython/cython/issues/4365')) @pytest.mark.skipif(pyximport is None, reason='cython is not installed') @pytest.mark.sphinx('html', testroot='ext-autodoc') def test_cython(app): From 98c53e03b71631e42767e7490ed94be1753c7cf2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 13 Sep 2023 03:00:54 +0100 Subject: [PATCH 4/6] KeyError --- sphinx/ext/autodoc/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index 781a1950a50..668d9fcd68e 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -110,7 +110,7 @@ def import_object(modname: str, objpath: list[str], objtype: str = '', try: typing.TYPE_CHECKING = True # Ignore failures; we've already successfully loaded these modules - with contextlib.suppress(ImportError): + with contextlib.suppress(ImportError, KeyError): for m in new_modules: _reload_module(sys.modules[m]) finally: From 6fb5d9cab1357fb9e67c89ccc2b0ac93442f3c25 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 13 Sep 2023 03:49:15 +0100 Subject: [PATCH 5/6] CHANGES --- CHANGES | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES b/CHANGES index 9c69efbc7b7..21cbe456edb 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,11 @@ Release 7.2.6 (in development) Bugs fixed ---------- +* #11679: Add the :envvar:`!SPHINX_AUTODOC_DO_NOT_RELOAD_MODULES` environment + variable, which if set skips reloading modules when using autodoc. + Patch by Matt Wozniski and Adam Turner. +* #11679: Use :py:func:`importlib.reload` to reload modules in autodoc. + Patch by Matt Wozniski and Adam Turner. Release 7.2.5 (released Aug 30, 2023) ===================================== From ec974cd019f33a8d1982532dc545eaa7ef6dff08 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 13 Sep 2023 20:18:44 +0100 Subject: [PATCH 6/6] Swap --- CHANGES | 4 ++-- doc/conf.py | 2 ++ sphinx/ext/autodoc/importer.py | 28 +++++++++++++--------------- tests/conftest.py | 3 +++ 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/CHANGES b/CHANGES index 21cbe456edb..89b0a8c2a63 100644 --- a/CHANGES +++ b/CHANGES @@ -4,8 +4,8 @@ Release 7.2.6 (in development) Bugs fixed ---------- -* #11679: Add the :envvar:`!SPHINX_AUTODOC_DO_NOT_RELOAD_MODULES` environment - variable, which if set skips reloading modules when using autodoc. +* #11679: Add the :envvar:`!SPHINX_AUTODOC_RELOAD_MODULES` environment variable, + which if set reloads modules when using autodoc with ``TYPE_CHECKING = True``. Patch by Matt Wozniski and Adam Turner. * #11679: Use :py:func:`importlib.reload` to reload modules in autodoc. Patch by Matt Wozniski and Adam Turner. diff --git a/doc/conf.py b/doc/conf.py index 85607681c21..d4915aa259d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -6,6 +6,8 @@ import sphinx +os.environ['SPHINX_AUTODOC_RELOAD_MODULES'] = '1' + extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.autosummary', 'sphinx.ext.extlinks', 'sphinx.ext.intersphinx', diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index 668d9fcd68e..84bfee58eb4 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -101,21 +101,19 @@ def import_object(modname: str, objpath: list[str], objtype: str = '', while module is None: try: original_module_names = frozenset(sys.modules) - import_module(modname, warningiserror=warningiserror) - - new_modules = [m for m in sys.modules if m not in original_module_names] - if os.environ.get('SPHINX_AUTODOC_DO_NOT_RELOAD_MODULES'): - new_modules = [] - # Try reloading modules with ``typing.TYPE_CHECKING == True``. - try: - typing.TYPE_CHECKING = True - # Ignore failures; we've already successfully loaded these modules - with contextlib.suppress(ImportError, KeyError): - for m in new_modules: - _reload_module(sys.modules[m]) - finally: - typing.TYPE_CHECKING = False - module = sys.modules[modname] + module = import_module(modname, warningiserror=warningiserror) + if os.environ.get('SPHINX_AUTODOC_RELOAD_MODULES'): + new_modules = [m for m in sys.modules if m not in original_module_names] + # Try reloading modules with ``typing.TYPE_CHECKING == True``. + try: + typing.TYPE_CHECKING = True + # Ignore failures; we've already successfully loaded these modules + with contextlib.suppress(ImportError, KeyError): + for m in new_modules: + _reload_module(sys.modules[m]) + finally: + typing.TYPE_CHECKING = False + module = sys.modules[modname] logger.debug('[autodoc] import %s => %r', modname, module) except ImportError as exc: logger.debug('[autodoc] import %s => failed', modname) diff --git a/tests/conftest.py b/tests/conftest.py index f0cc23e71a8..1b909bded02 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import os from pathlib import Path import docutils @@ -24,6 +25,8 @@ def _init_console(locale_dir=sphinx.locale._LOCALE_DIR, catalog='sphinx'): # Exclude 'roots' dirs for pytest test collector collect_ignore = ['roots'] +os.environ['SPHINX_AUTODOC_RELOAD_MODULES'] = '1' + @pytest.fixture(scope='session') def rootdir():