[8.x] Parser agnostic i18n Locale transform

sphinx/io.py

@@ -143,6 +143,11 @@ def setup(self, app: Sphinx) -> None:
             if transform in self.transforms:
                 self.transforms.remove(transform)
 
+    def parse(self) -> None:
+        """Override the BaseReader parse method to call self.parser.parse_inline()."""
+        self.document = document = self.new_document()
+        self.parser.parse_inline(self.input, document, 1)
+
 
 class SphinxDummyWriter(UnfilteredWriter):
     """Dummy writer module used for generating doctree."""

sphinx/parsers.py

@@ -7,7 +7,7 @@
 import docutils.parsers
 import docutils.parsers.rst
 from docutils import nodes
-from docutils.parsers.rst import states
+from docutils.parsers.rst import languages, states
 from docutils.statemachine import StringList
 from docutils.transforms.universal import SmartQuotes
 
@@ -46,6 +46,11 @@ def set_application(self, app: Sphinx) -> None:
         self.config = app.config
         self.env = app.env
 
+    def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) -> None:
+        """Parse the inline elements of a text block and generate a document tree."""
+        msg = 'Parser subclasses must implement parse_inline'
+        raise NotImplementedError(msg)
+
 
 class RSTParser(docutils.parsers.rst.Parser, Parser):
     """A reST parser for Sphinx."""
@@ -60,6 +65,30 @@ def get_transforms(self) -> list[type[Transform]]:
         transforms.remove(SmartQuotes)
         return transforms
 
+    def parse_inline(self, inputstring: str, document: nodes.document, lineno: int) -> None:
+        """Parse inline syntax from text and generate a document tree."""
+        # Avoid "Literal block expected; none found." warnings.
+        if inputstring.endswith('::'):
+            inputstring = inputstring[:-1]
+
+        reporter = document.reporter
+        reporter.get_source_and_line = lambda x: (document['source'], x)  # type: ignore[attr-defined]
+        language = languages.get_language(document.settings.language_code, reporter)
+        if self.inliner is None:
+            inliner = states.Inliner()
+        else:
+            inliner = self.inliner
+        inliner.init_customizations(document.settings)
+        memo = states.Struct(
+            document=document,
+            reporter=reporter,
+            language=language,
+        )
+        textnodes, messages = inliner.parse(inputstring, lineno, memo, document)
+        p = nodes.paragraph(inputstring, '', *textnodes)
+        p.source, p.line = document['source'], lineno
+        document += [p, *messages]
+
     def parse(self, inputstring: str | StringList, document: nodes.document) -> None:
         """Parse text and generate a document tree."""
         self.setup_parse(inputstring, document)  # type: ignore[arg-type]

sphinx/transforms/i18n.py

@@ -5,7 +5,6 @@
 import contextlib
 from os import path
 from re import DOTALL, match
-from textwrap import indent
 from typing import TYPE_CHECKING, Any, TypeVar
 
 from docutils import nodes
@@ -21,7 +20,6 @@
 from sphinx.util.i18n import docname_to_domain
 from sphinx.util.index_entries import split_index_msg
 from sphinx.util.nodes import (
-    IMAGE_TYPE_NODES,
     LITERAL_TYPE_NODES,
     NodeMatcher,
     extract_messages,
@@ -380,25 +378,12 @@ def apply(self, **kwargs: Any) -> None:
                 node['translated'] = True
                 continue
 
-            # Avoid "Literal block expected; none found." warnings.
-            # If msgstr ends with '::' then it cause warning message at
-            # parser.parse() processing.
-            # literal-block-warning is only appear in avobe case.
-            if msgstr.strip().endswith('::'):
-                msgstr += '\n\n   dummy literal'
-                # dummy literal node will discard by 'patch = patch[0]'
-
-            # literalblock need literal block notation to avoid it become
-            # paragraph.
+            # literalblock can not contain references or terms
             if isinstance(node, LITERAL_TYPE_NODES):
-                msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
+                continue
 
             patch = publish_msgstr(self.app, msgstr, source,
                                    node.line, self.config, settings)  # type: ignore[arg-type]
-            # FIXME: no warnings about inconsistent references in this part
-            # XXX doctest and other block markup
-            if not isinstance(patch, nodes.paragraph):
-                continue  # skip for now
 
             updater = _NodeUpdater(node, patch, self.document, noqa=False)
             processed = updater.update_title_mapping()
@@ -453,45 +438,25 @@ def apply(self, **kwargs: Any) -> None:
                 node['alt'] = msgstr
                 continue
 
-            # Avoid "Literal block expected; none found." warnings.
-            # If msgstr ends with '::' then it cause warning message at
-            # parser.parse() processing.
-            # literal-block-warning is only appear in avobe case.
-            if msgstr.strip().endswith('::'):
-                msgstr += '\n\n   dummy literal'
-                # dummy literal node will discard by 'patch = patch[0]'
-
-            # literalblock need literal block notation to avoid it become
-            # paragraph.
-            if isinstance(node, LITERAL_TYPE_NODES):
-                msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
+            if isinstance(node, nodes.image) and node.get('uri') == msg:
+                node['uri'] = msgstr
+                continue
 
-            # Structural Subelements phase1
-            # There is a possibility that only the title node is created.
-            # see: https://docutils.sourceforge.io/docs/ref/doctree.html#structural-subelements
-            if isinstance(node, nodes.title):
-                # This generates: <section ...><title>msgstr</title></section>
-                msgstr = msgstr + '\n' + '=' * len(msgstr) * 2
+            # literalblock do not need to be parsed as they do not contain inline syntax,
+            # except for parsed-literals, but they use the same node type, so we differentiate
+            # them based on their number of children.
+            if isinstance(node, LITERAL_TYPE_NODES) and len(node.children) <= 1:
+                node.children = [nodes.Text(msgstr)]
+                # for highlighting that expects .rawsource and .astext() are same.
+                node.rawsource = node.astext()
+                node['translated'] = True
+                continue
 
             patch = publish_msgstr(self.app, msgstr, source,
                                    node.line, self.config, settings)  # type: ignore[arg-type]
-            # Structural Subelements phase2
-            if isinstance(node, nodes.title):
-                # get <title> node that placed as a first child
-                patch = patch.next_node()  # type: ignore[assignment]
 
             # ignore unexpected markups in translation message
-            unexpected: tuple[type[nodes.Element], ...] = (
-                nodes.paragraph,    # expected form of translation
-                nodes.title,        # generated by above "Subelements phase2"
-            )
-
-            # following types are expected if
-            # config.gettext_additional_targets is configured
-            unexpected += LITERAL_TYPE_NODES
-            unexpected += IMAGE_TYPE_NODES
-
-            if not isinstance(patch, unexpected):
+            if not isinstance(patch, nodes.paragraph):
                 continue  # skip
 
             updater = _NodeUpdater(node, patch, self.document, noqa)
@@ -502,15 +467,6 @@ def apply(self, **kwargs: Any) -> None:
             updater.update_pending_xrefs()
             updater.update_leaves()
 
-            # for highlighting that expects .rawsource and .astext() are same.
-            if isinstance(node, LITERAL_TYPE_NODES):
-                node.rawsource = node.astext()
-
-            if isinstance(node, nodes.image) and node.get('alt') != msg:
-                node['uri'] = patch['uri']
-                node['translated'] = False
-                continue  # do not mark translated
-
             node['translated'] = True  # to avoid double translation
 
         if 'index' in self.config.gettext_additional_targets:

sphinx/util/nodes.py

@@ -266,8 +266,7 @@ def extract_messages(doctree: Element) -> Iterable[tuple[Element, str]]:
             if node.get('alt'):
                 yield node, node['alt']
             if node.get('translatable'):
-                image_uri = node.get('original_uri', node['uri'])
-                msg = f'.. image:: {image_uri}'
+                msg = node.get('original_uri', node['uri'])
             else:
                 msg = ''
         elif isinstance(node, nodes.meta):

tests/roots/test-intl/literalblock.txt

@@ -69,3 +69,11 @@ doctest blocks
 >>> if __name__ == '__main__':  # if run this py file as python script
 ...     main()  # call main
 
+
+parsed literal
+==============
+
+.. parsed-literal::
+
+   **this** *is*
+   `parsed literal`_

tests/roots/test-intl/markup.txt

@@ -4,3 +4,5 @@ i18n with strange markup
 1. title starting with 1.
 -------------------------
 
+A. Einstein was a really
+smart dude.

tests/roots/test-intl/xx/LC_MESSAGES/figure.po

@@ -40,14 +40,14 @@ msgstr "IMAGE URL AND ALT"
 msgid "img"
 msgstr "IMG -> I18N"
 
-msgid ".. image:: img.png"
-msgstr ".. image:: i18n.png"
+msgid "img.png"
+msgstr "i18n.png"
 
 msgid "i18n"
 msgstr "I18N -> IMG"
 
-msgid ".. image:: i18n.png"
-msgstr ".. image:: img.png"
+msgid "i18n.png"
+msgstr "img.png"
 
 msgid "image on substitution"
 msgstr "IMAGE ON SUBSTITUTION"

tests/roots/test-intl/xx/LC_MESSAGES/literalblock.po

@@ -125,3 +125,13 @@ msgstr ""
 ">>> if __name__ == '__main__':  # IF RUN THIS PY FILE AS PYTHON SCRIPT\n"
 "...     main()  # CALL MAIN"
 
+msgid "parsed literal"
+msgstr "PARSED LITERAL"
+
+msgid ""
+"**this** *is*\n"
+"`parsed literal`_"
+msgstr ""
+"**THIS** *IS*\n"
+"`PARSED LITERAL`_"
+

tests/roots/test-intl/xx/LC_MESSAGES/markup.po

@@ -23,3 +23,6 @@ msgstr "I18N WITH STRANGE MARKUP"
 msgid "1. title starting with 1."
 msgstr "1. TITLE STARTING WITH 1."
 
+msgid "A. Einstein was a really smart dude."
+msgstr "A. EINSTEIN WAS A REALLY SMART DUDE."
+

tests/roots/test-intl_substitution_definitions/xx/LC_MESSAGES/prolog_epilog_substitution.po

@@ -28,11 +28,11 @@ msgstr "SUBSTITUTED IMAGE |subst_epilog_2| HERE."
 msgid "subst_prolog_2"
 msgstr "SUBST_PROLOG_2 TRANSLATED"
 
-msgid ".. image:: /img.png"
-msgstr ".. image:: /i18n.png"
+msgid "/img.png"
+msgstr "/i18n.png"
 
 msgid "subst_epilog_2"
 msgstr "SUBST_EPILOG_2 TRANSLATED"
 
-msgid ".. image:: /i18n.png"
-msgstr ".. image:: /img.png"
+msgid "/i18n.png"
+msgstr "/img.png"

tests/test_builders/test_build_gettext.py

@@ -206,11 +206,11 @@ def test_gettext_prolog_epilog_substitution(app):
         "This is content that contains |subst_prolog_1|.",
         "Substituted image |subst_prolog_2| here.",
         "subst_prolog_2",
-        ".. image:: /img.png",
+        "/img.png",
         "This is content that contains |subst_epilog_1|.",
         "Substituted image |subst_epilog_2| here.",
         "subst_epilog_2",
-        ".. image:: /i18n.png",
+        "/i18n.png",
     ]
 
 
@@ -265,4 +265,6 @@ def test_gettext_literalblock_additional(app):
         "function\\n...     sys.stdout.write('hello')  # call write method of "
         "stdout object\\n>>>\\n>>> if __name__ == '__main__':  # if run this py "
         'file as python script\\n...     main()  # call main',
+        'parsed literal',
+        '**this** *is*\\n`parsed literal`_',
     ]