Add content_offset parameter to nested_parse_with_titles

Previously, `nested_parse_with_titles` always passed `0` as the input
offset when invoking `nested_parse`.  When parsing the content of a
directive, as is a common use case for `nested_parse_with_titles`,
this leads to incorrect source file/line number information, as it
does not take into account the directive's `content_offset`, which is
always non-zero.

This issue affects *all* object descriptions due to #10887.  It also
affects the `ifconfig` extension.

The `py:module` and `js:module` directives employed a workaround for
this issue, by wrapping the calls to `nested_parse_with_title` with
`switch_source_input`.  That worked, but was more complicated (and
likely less efficient) than necessary.

This commit adds an optional `content_offset` parameter to
`nested_parse_with_titles`, and fixes callers to pass the appropriate
content offset when needed.

This commit eliminates the now-unnecessary calls to
`switch_source_input` and instead specifies the correct `content_offset`.

sphinx/directives/__init__.py

@@ -262,7 +262,7 @@ def run(self) -> list[Node]:
             # needed for association of version{added,changed} directives
             self.env.temp_data['object'] = self.names[0]
         self.before_content()
-        nested_parse_with_titles(self.state, self.content, contentnode)
+        nested_parse_with_titles(self.state, self.content, contentnode, self.content_offset)
         self.transform_content(contentnode)
         self.env.app.emit('object-description-transform',
                           self.domain, self.objtype, contentnode)

sphinx/domains/javascript.py

@@ -20,7 +20,7 @@
 from sphinx.roles import XRefRole
 from sphinx.util import logging
 from sphinx.util.docfields import Field, GroupedField, TypedField
-from sphinx.util.docutils import SphinxDirective, switch_source_input
+from sphinx.util.docutils import SphinxDirective
 from sphinx.util.nodes import make_id, make_refnode, nested_parse_with_titles
 from sphinx.util.typing import OptionSpec
 
@@ -297,10 +297,9 @@ def run(self) -> list[Node]:
         noindex = 'noindex' in self.options
 
         content_node: Element = nodes.section()
-        with switch_source_input(self.state, self.content):
-            # necessary so that the child nodes get the right source/line set
-            content_node.document = self.state.document
-            nested_parse_with_titles(self.state, self.content, content_node)
+        # necessary so that the child nodes get the right source/line set
+        content_node.document = self.state.document
+        nested_parse_with_titles(self.state, self.content, content_node, self.content_offset)
 
         ret: list[Node] = []
         if not noindex:

sphinx/domains/python.py

@@ -26,7 +26,7 @@
 from sphinx.roles import XRefRole
 from sphinx.util import logging
 from sphinx.util.docfields import Field, GroupedField, TypedField
-from sphinx.util.docutils import SphinxDirective, switch_source_input
+from sphinx.util.docutils import SphinxDirective
 from sphinx.util.inspect import signature_from_str
 from sphinx.util.nodes import (
     find_pending_xref_condition,
@@ -1034,10 +1034,9 @@ def run(self) -> list[Node]:
         self.env.ref_context['py:module'] = modname
 
         content_node: Element = nodes.section()
-        with switch_source_input(self.state, self.content):
-            # necessary so that the child nodes get the right source/line set
-            content_node.document = self.state.document
-            nested_parse_with_titles(self.state, self.content, content_node)
+        # necessary so that the child nodes get the right source/line set
+        content_node.document = self.state.document
+        nested_parse_with_titles(self.state, self.content, content_node, self.content_offset)
 
         ret: list[Node] = []
         if not noindex:

sphinx/ext/autodoc/__init__.py

@@ -312,7 +312,7 @@ class Documenter:
     #: order if autodoc_member_order is set to 'groupwise'
     member_order = 0
     #: true if the generated content may contain titles
-    titles_allowed = False
+    titles_allowed = True
 
     option_spec: OptionSpec = {
         'noindex': bool_option
@@ -960,7 +960,6 @@ class ModuleDocumenter(Documenter):
     """
     objtype = 'module'
     content_indent = ''
-    titles_allowed = True
     _extra_indent = '   '
 
     option_spec: OptionSpec = {

sphinx/ext/ifconfig.py

@@ -45,7 +45,7 @@ def run(self) -> list[Node]:
         node.document = self.state.document
         self.set_source_info(node)
         node['expr'] = self.arguments[0]
-        nested_parse_with_titles(self.state, self.content, node)
+        nested_parse_with_titles(self.state, self.content, node, self.content_offset)
         return [node]
 
 

sphinx/util/nodes.py

@@ -311,7 +311,8 @@ def traverse_translatable_index(
         yield node, entries
 
 
-def nested_parse_with_titles(state: Any, content: StringList, node: Node) -> str:
+def nested_parse_with_titles(state: Any, content: StringList, node: Node,
+                             content_offset: int = 0) -> str:
     """Version of state.nested_parse() that allows titles and does not require
     titles to have the same decoration as the calling document.
 
@@ -324,7 +325,7 @@ def nested_parse_with_titles(state: Any, content: StringList, node: Node) -> str
     state.memo.title_styles = []
     state.memo.section_level = 0
     try:
-        return state.nested_parse(content, 0, node, match_titles=1)
+        return state.nested_parse(content, content_offset, node, match_titles=1)
     finally:
         state.memo.title_styles = surrounding_title_styles
         state.memo.section_level = surrounding_section_level

tests/test_directive_object_description.py

@@ -2,9 +2,11 @@
 
 import pytest
 from docutils import nodes
+import docutils.utils
 
 from sphinx import addnodes
 from sphinx.io import create_publisher
+from sphinx.testing import restructuredtext
 from sphinx.util.docutils import sphinx_domains
 
 
@@ -43,3 +45,15 @@ def test_object_description_sections(app):
     assert doctree[1][1][0][0][0] == 'Overview'
     assert isinstance(doctree[1][1][0][1], nodes.paragraph)
     assert doctree[1][1][0][1][0] == 'Lorem ipsum dolar sit amet'
+
+
+def test_object_description_content_line_number(app):
+    text = (".. py:function:: foo(bar)\n" +
+            "\n" +
+            "   Some link here: :ref:`abc`\n")
+    doc = restructuredtext.parse(app, text)
+    xrefs = list(doc.findall(condition=addnodes.pending_xref))
+    assert len(xrefs) == 1
+    source, line = docutils.utils.get_source_line(xrefs[0])
+    assert 'index.rst' in source
+    assert line == 3

tests/test_domain_js.py

@@ -4,6 +4,7 @@
 
 import pytest
 from docutils import nodes
+import docutils.utils
 
 from sphinx import addnodes
 from sphinx.addnodes import (
@@ -229,3 +230,15 @@ def test_noindexentry(app):
     assert_node(doctree, (addnodes.index, desc, addnodes.index, desc))
     assert_node(doctree[0], addnodes.index, entries=[('single', 'f() (built-in function)', 'f', '', None)])
     assert_node(doctree[2], addnodes.index, entries=[])
+
+
+def test_module_content_line_number(app):
+    text = (".. js:module:: foo\n" +
+            "\n" +
+            "   Some link here: :ref:`abc`\n")
+    doc = restructuredtext.parse(app, text)
+    xrefs = list(doc.findall(condition=addnodes.pending_xref))
+    assert len(xrefs) == 1
+    source, line = docutils.utils.get_source_line(xrefs[0])
+    assert 'index.rst' in source
+    assert line == 3

tests/test_domain_py.py

@@ -1458,3 +1458,15 @@ def test_signature_line_number(app, include_options):
     source, line = docutils.utils.get_source_line(xrefs[0])
     assert 'index.rst' in source
     assert line == 1
+
+
+def test_module_content_line_number(app):
+    text = (".. py:module:: foo\n" +
+            "\n" +
+            "   Some link here: :ref:`abc`\n")
+    doc = restructuredtext.parse(app, text)
+    xrefs = list(doc.findall(condition=addnodes.pending_xref))
+    assert len(xrefs) == 1
+    source, line = docutils.utils.get_source_line(xrefs[0])
+    assert 'index.rst' in source
+    assert line == 3

tests/test_ext_ifconfig.py

@@ -1,6 +1,9 @@
 """Test sphinx.ext.ifconfig extension."""
 
+import docutils.utils
 import pytest
+from sphinx import addnodes
+from sphinx.testing import restructuredtext
 
 
 @pytest.mark.sphinx('text', testroot='ext-ifconfig')
@@ -9,3 +12,16 @@ def test_ifconfig(app, status, warning):
     result = (app.outdir / 'index.txt').read_text(encoding='utf8')
     assert 'spam' in result
     assert 'ham' not in result
+
+
+def test_ifconfig_content_line_number(app):
+    app.setup_extension("sphinx.ext.ifconfig")
+    text = (".. ifconfig:: confval1\n" +
+            "\n" +
+            "   Some link here: :ref:`abc`\n")
+    doc = restructuredtext.parse(app, text)
+    xrefs = list(doc.findall(condition=addnodes.pending_xref))
+    assert len(xrefs) == 1
+    source, line = docutils.utils.get_source_line(xrefs[0])
+    assert 'index.rst' in source
+    assert line == 3