Add a new references builder

.ruff.toml

@@ -428,7 +428,21 @@ quote-style = "single"
 exclude = [
     "sphinx/addnodes.py",
     "sphinx/application.py",
-    "sphinx/builders/**/*",
+    "sphinx/builders/__init__.py",
+    "sphinx/builders/_epub_base.py",
+    "sphinx/builders/changes.py",
+    "sphinx/builders/dirhtml.py",
+    "sphinx/builders/dummy.py",
+    "sphinx/builders/epub3.py",
+    "sphinx/builders/gettext.py",
+    "sphinx/builders/html/**/*",
+    "sphinx/builders/latex/**/*",
+    "sphinx/builders/linkcheck.py",
+    "sphinx/builders/manpage.py",
+    "sphinx/builders/singlehtml.py",
+    "sphinx/builders/texinfo.py",
+    "sphinx/builders/text.py",
+    "sphinx/builders/xml.py",
     "sphinx/cmd/**/*",
     "sphinx/config.py",
     "sphinx/directives/**/*",

sphinx/application.py

@@ -65,6 +65,7 @@
     'sphinx.builders.latex',
     'sphinx.builders.linkcheck',
     'sphinx.builders.manpage',
+    'sphinx.builders.references',
     'sphinx.builders.singlehtml',
     'sphinx.builders.texinfo',
     'sphinx.builders.text',

sphinx/builders/references.py

@@ -0,0 +1,148 @@
+"""A builder that creates a lookup for available references in the project."""
+
+from __future__ import annotations
+
+import json
+from os import path
+from typing import TYPE_CHECKING, Iterator, Literal, TypedDict
+
+from sphinx.builders import Builder
+from sphinx.ext.intersphinx import InventoryAdapter
+from sphinx.locale import __
+from sphinx.util.display import progress_message
+
+if TYPE_CHECKING:
+    from docutils.nodes import Node
+
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
+
+
+class LocalReference(TypedDict, total=False):
+    type: Literal['local']
+    document: str
+    """The path to the document."""
+    dispname: str
+    """The implicit display text."""
+
+
+class RemoteReference(TypedDict, total=False):
+    type: Literal['remote']
+    key: str
+    """The ``intersphinx_mapping`` key."""
+    url: str
+    """The full URL to this target."""
+    dispname: str
+    """The implicit display text."""
+
+
+class ReferenceTargets(TypedDict, total=False):
+    items: dict[str, list[LocalReference | RemoteReference]]
+    """Mapping of the target name to its data."""
+    roles: list[str]
+    """List of available roles for this object type."""
+
+
+ReferenceJson = dict[str, dict[str, ReferenceTargets]]
+"""A mapping of ``domain name`` -> ``object type`` -> ``targets``."""
+
+
+class ReferencesBuilder(Builder):
+    name = 'references'
+    epilog = __('Generated reference lookup at %(outdir)s/references.json.')
+
+    allow_parallel = True
+
+    def init(self) -> None:
+        pass
+
+    def get_outdated_docs(self) -> Iterator[str]:
+        for docname in self.env.found_docs:
+            if docname not in self.env.all_docs:
+                yield docname
+                continue
+            targetname = self.doctreedir.joinpath(docname + '.doctree')
+            try:
+                targetmtime = path.getmtime(targetname)
+            except Exception:
+                targetmtime = 0
+            try:
+                srcmtime = path.getmtime(self.env.doc2path(docname))
+                if srcmtime > targetmtime:
+                    yield docname
+            except OSError:
+                # source doesn't exist anymore
+                pass
+
+    def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+        return ''
+
+    def prepare_writing(self, docnames: set[str]) -> None:
+        pass
+
+    def write_doc(self, docname: str, doctree: Node) -> None:
+        pass
+
+    def finish(self) -> None:
+        self.finish_tasks.add_task(self.write_references)
+
+    @progress_message(__('writing references'))
+    def write_references(self) -> None:
+        data: ReferenceJson = {}
+        # local objects
+        for domainname, domain in self.env.domains.items():
+            for name, dispname, otype_name, docname, _, _ in domain.get_objects():
+                dispname = str(dispname)  # can be a _TranslationProxy object
+                local_data: LocalReference = {
+                    'type': 'local',
+                    'document': self.env.doc2path(docname),
+                }
+                # only add dispname if it is set and not the same as name
+                if not (dispname == name or not dispname or dispname == '-'):
+                    local_data['dispname'] = dispname
+                # record available roles for this object type, if available
+                if otype_name not in data.setdefault(domainname, {}):
+                    data[domainname][otype_name] = {'items': {}}
+                    if otype := domain.object_types.get(otype_name):
+                        data[domainname][otype_name]['roles'] = list(otype.roles)
+                data.setdefault(domainname, {}).setdefault(otype_name, {})['items'].setdefault(
+                    name, []
+                ).append(local_data)
+        # remote objects (if using intersphinx)
+        inventories = InventoryAdapter(self.env)
+        for inv_key, invdata in inventories.named_inventory.items():
+            for domain_oject, names in invdata.items():
+                domainname, otype_name = domain_oject.split(':', 1)
+                local_domain = self.env.domains.get(domainname)
+                for name, (_, _, url, dispname) in names.items():
+                    remote_data: RemoteReference = {
+                        'type': 'remote',
+                        'key': inv_key,
+                        'url': url,
+                    }
+                    # only add dispname if it is set and not the same as name
+                    if not (dispname == name or not dispname or dispname == '-'):
+                        remote_data['dispname'] = dispname
+                    if otype_name not in data.setdefault(domainname, {}):
+                        data[domainname][otype_name] = {'items': {}}
+                        # record available roles for this object type, if available
+                        if local_domain and (
+                            otype := local_domain.object_types.get(otype_name)
+                        ):
+                            data[domainname][otype_name]['roles'] = list(otype.roles)
+                    data.setdefault(domainname, {}).setdefault(otype_name, {})[
+                        'items'
+                    ].setdefault(name, []).append(remote_data)
+
+        with self.outdir.joinpath('references.json').open('w', encoding='utf-8') as fp:
+            json.dump(data, fp, sort_keys=True, indent=2)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    app.add_builder(ReferencesBuilder)
+
+    return {
+        'version': 'builtin',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

tests/roots/test-build-refs-with-intersphinx/conf.py

@@ -0,0 +1,4 @@
+extensions = ['sphinx.ext.intersphinx']
+intersphinx_mapping = {
+    'key': ('https://example.com', 'objects.inv')
+}

tests/roots/test-build-refs-with-intersphinx/index.rst

@@ -0,0 +1,31 @@
+The basic Sphinx documentation for testing
+==========================================
+
+Sphinx is a tool that makes it easy to create intelligent and beautiful
+documentation for Python projects (or other documents consisting of multiple
+reStructuredText sources), written by Georg Brandl.  It was originally created
+for the new Python documentation, and has excellent facilities for Python
+project documentation, but C/C++ is supported as well, and more languages are
+planned.
+
+Sphinx uses reStructuredText as its markup language, and many of its strengths
+come from the power and straightforwardness of reStructuredText and its parsing
+and translating suite, the Docutils.
+
+features
+--------
+
+Among its features are the following:
+
+* Output formats: HTML (including derivative formats such as HTML Help, Epub
+  and Qt Help), plain text, manual pages and LaTeX or direct PDF output
+  using rst2pdf
+* Extensive cross-references: semantic markup and automatic links
+  for functions, classes, glossary terms and similar pieces of information
+* Hierarchical structure: easy definition of a document tree, with automatic
+  links to siblings, parents and children
+* Automatic indices: general index as well as a module index
+* Code handling: automatic highlighting using the Pygments highlighter
+* Flexible HTML output using the Jinja 2 templating engine
+* Various extensions are available, e.g. for automatic testing of snippets
+  and inclusion of appropriately formatted docstrings

tests/test_builders/test_build_references.py

@@ -0,0 +1,155 @@
+"""Test the build process of the references builder."""
+
+import json
+
+import pytest
+
+
+@pytest.mark.sphinx('references', testroot='basic')
+def test_basic(app):
+    """Test for a basic document build."""
+    app.build()
+    # there should be no warnings
+    assert not app.warning.getvalue()
+    # there should be only one file in the output
+    assert [p.relative_to(app.outdir).as_posix() for p in app.outdir.iterdir()] == [
+        'references.json'
+    ]
+    # test the content of the reference file
+    content = (app.outdir / 'references.json').read_text('utf-8')
+    data = json.loads(content)
+    assert data == {
+        'std': {
+            'doc': {
+                'items': {
+                    'index': [
+                        {
+                            'dispname': 'The basic Sphinx documentation for testing',
+                            'document': str(app.srcdir / 'index.rst'),
+                            'type': 'local',
+                        }
+                    ]
+                },
+                'roles': ['doc'],
+            },
+            'label': {
+                'items': {
+                    'genindex': [
+                        {
+                            'dispname': 'Index',
+                            'document': str(app.srcdir / 'genindex.rst'),
+                            'type': 'local',
+                        }
+                    ],
+                    'modindex': [
+                        {
+                            'dispname': 'Module Index',
+                            'document': str(app.srcdir / 'py-modindex.rst'),
+                            'type': 'local',
+                        }
+                    ],
+                    'py-modindex': [
+                        {
+                            'dispname': 'Python Module Index',
+                            'document': str(app.srcdir / 'py-modindex.rst'),
+                            'type': 'local',
+                        }
+                    ],
+                    'search': [
+                        {
+                            'dispname': 'Search Page',
+                            'document': str(app.srcdir / 'search.rst'),
+                            'type': 'local',
+                        }
+                    ],
+                },
+                'roles': ['ref', 'keyword'],
+            },
+        }
+    }
+
+
+@pytest.mark.sphinx('references', testroot='build-refs-with-intersphinx')
+def test_with_intersphinx(app):
+    """Test for a document built with an intersphinx mapping configured."""
+    app.build()
+    # there should be no warnings
+    assert not app.warning.getvalue()
+    # there should be only one file in the output
+    assert [p.relative_to(app.outdir).as_posix() for p in app.outdir.iterdir()] == [
+        'references.json'
+    ]
+    # test the content of the reference file
+    content = (app.outdir / 'references.json').read_text('utf-8')
+    data = json.loads(content)
+    assert data == {
+        'std': {
+            'doc': {
+                'items': {
+                    'index': [
+                        {
+                            'dispname': 'The basic Sphinx documentation for testing',
+                            'document': str(app.srcdir / 'index.rst'),
+                            'type': 'local',
+                        }
+                    ]
+                },
+                'roles': ['doc'],
+            },
+            'label': {
+                'items': {
+                    'genindex': [
+                        {
+                            'dispname': 'Index',
+                            'document': str(app.srcdir / 'genindex.rst'),
+                            'type': 'local',
+                        }
+                    ],
+                    'modindex': [
+                        {
+                            'dispname': 'Module Index',
+                            'document': str(app.srcdir / 'py-modindex.rst'),
+                            'type': 'local',
+                        }
+                    ],
+                    'py-modindex': [
+                        {
+                            'dispname': 'Python Module Index',
+                            'document': str(app.srcdir / 'py-modindex.rst'),
+                            'type': 'local',
+                        }
+                    ],
+                    'search': [
+                        {
+                            'dispname': 'Search Page',
+                            'document': str(app.srcdir / 'search.rst'),
+                            'type': 'local',
+                        }
+                    ],
+                },
+                'roles': ['ref', 'keyword'],
+            },
+        },
+        'py': {
+            'module': {
+                'items': {
+                    'module1': [
+                        {
+                            'dispname': 'Long Module desc',
+                            'key': 'key',
+                            'type': 'remote',
+                            'url': 'https://example.com/foo.html#module-module1',
+                        }
+                    ],
+                    'module2': [
+                        {
+                            'key': 'key',
+                            'type': 'remote',
+                            'url': 'https://example.com/foo.html#module-module2',
+                        }
+                    ],
+                },
+                'roles': ['mod', 'obj'],
+            }
+        },
+    }