Allow attr_list quoted values to contain curly braces

.spell-dict

@@ -146,6 +146,7 @@ Treeprocessor
 Treeprocessors
 tuple
 tuples
+unparsable
 unclosed
 unescape
 unescaping

docs/changelog.md

@@ -34,8 +34,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Include `scripts/*.py` in the generated source tarballs (#1430).
 * Ensure lines after heading in loose list are properly detabbed (#1443).
 * Give smarty tree processor higher priority than toc (#1440).
-* Permit carrots (`^`) and square brackets (`]`) but explicitly exclude
+* Permit carets (`^`) and square brackets (`]`) but explicitly exclude
   backslashes (`\`) from abbreviations (#1444).
+* In attribute lists (`attr_list`, `fenced_code`), quoted attribute values are
+  now allowed to contain curly braces (`}`) (#1414).
 
 ## [3.5.2] -- 2024-01-10
 

markdown/extensions/attr_list.py

@@ -57,17 +57,30 @@ def _handle_word(s, t):
 
 
 _scanner = re.Scanner([
-    (r'[^ =]+=".*?"', _handle_double_quote),
-    (r"[^ =]+='.*?'", _handle_single_quote),
-    (r'[^ =]+=[^ =]+', _handle_key_value),
-    (r'[^ =]+', _handle_word),
+    (r'[^ =}]+=".*?"', _handle_double_quote),
+    (r"[^ =}]+='.*?'", _handle_single_quote),
+    (r'[^ =}]+=[^ =}]+', _handle_key_value),
+    (r'[^ =}]+', _handle_word),
     (r' ', None)
 ])
 
 
-def get_attrs(str: str) -> list[tuple[str, str]]:
-    """ Parse attribute list and return a list of attribute tuples. """
-    return _scanner.scan(str)[0]
+def get_attrs_and_remainder(attrs_string: str) -> tuple[list[tuple[str, str]], str]:
+    """ Parse attribute list and return a list of attribute tuples.
+
+    Additionally, return any text that remained after a curly brace. In typical cases, its presence
+    should mean that the input does not match the intended attribute list syntax.
+    """
+    attrs, remainder = _scanner.scan(attrs_string)
+    # To keep historic behavior, discard all unparsable text prior to '}'.
+    index = remainder.find('}')
+    remainder = remainder[index:] if index != -1 else ''
+    return attrs, remainder
+
+
+def get_attrs(str: str) -> list[tuple[str, str]]:  # pragma: no cover
+    """ Soft-deprecated. Prefer `get_attrs_and_remainder`. """
+    return get_attrs_and_remainder(str)[0]
 
 
 def isheader(elem: Element) -> bool:
@@ -76,7 +89,7 @@ def isheader(elem: Element) -> bool:
 
 class AttrListTreeprocessor(Treeprocessor):
 
-    BASE_RE = r'\{\:?[ ]*([^\}\n ][^\}\n]*)[ ]*\}'
+    BASE_RE = r'\{\:?[ ]*([^\}\n ][^\n]*)[ ]*\}'
     HEADER_RE = re.compile(r'[ ]+{}[ ]*$'.format(BASE_RE))
     BLOCK_RE = re.compile(r'\n[ ]*{}[ ]*$'.format(BASE_RE))
     INLINE_RE = re.compile(r'^{}'.format(BASE_RE))
@@ -106,49 +119,58 @@ def run(self, doc: Element) -> None:
                         # use tail of last child. no `ul` or `ol`.
                         m = RE.search(elem[-1].tail)
                         if m:
-                            self.assign_attrs(elem, m.group(1))
-                            elem[-1].tail = elem[-1].tail[:m.start()]
+                            if not self.assign_attrs(elem, m.group(1), strict=True):
+                                elem[-1].tail = elem[-1].tail[:m.start()]
                     elif pos is not None and pos > 0 and elem[pos-1].tail:
                         # use tail of last child before `ul` or `ol`
                         m = RE.search(elem[pos-1].tail)
                         if m:
-                            self.assign_attrs(elem, m.group(1))
-                            elem[pos-1].tail = elem[pos-1].tail[:m.start()]
+                            if not self.assign_attrs(elem, m.group(1), strict=True):
+                                elem[pos-1].tail = elem[pos-1].tail[:m.start()]
                     elif elem.text:
                         # use text. `ul` is first child.
                         m = RE.search(elem.text)
                         if m:
-                            self.assign_attrs(elem, m.group(1))
-                            elem.text = elem.text[:m.start()]
+                            if not self.assign_attrs(elem, m.group(1), strict=True):
+                                elem.text = elem.text[:m.start()]
                 elif len(elem) and elem[-1].tail:
                     # has children. Get from tail of last child
                     m = RE.search(elem[-1].tail)
                     if m:
-                        self.assign_attrs(elem, m.group(1))
-                        elem[-1].tail = elem[-1].tail[:m.start()]
-                        if isheader(elem):
-                            # clean up trailing #s
-                            elem[-1].tail = elem[-1].tail.rstrip('#').rstrip()
+                        if not self.assign_attrs(elem, m.group(1), strict=True):
+                            elem[-1].tail = elem[-1].tail[:m.start()]
+                            if isheader(elem):
+                                # clean up trailing #s
+                                elem[-1].tail = elem[-1].tail.rstrip('#').rstrip()
                 elif elem.text:
                     # no children. Get from text.
                     m = RE.search(elem.text)
                     if m:
-                        self.assign_attrs(elem, m.group(1))
-                        elem.text = elem.text[:m.start()]
-                        if isheader(elem):
-                            # clean up trailing #s
-                            elem.text = elem.text.rstrip('#').rstrip()
+                        if not self.assign_attrs(elem, m.group(1), strict=True):
+                            elem.text = elem.text[:m.start()]
+                            if isheader(elem):
+                                # clean up trailing #s
+                                elem.text = elem.text.rstrip('#').rstrip()
             else:
                 # inline: check for `attrs` at start of tail
                 if elem.tail:
                     m = self.INLINE_RE.match(elem.tail)
                     if m:
-                        self.assign_attrs(elem, m.group(1))
-                        elem.tail = elem.tail[m.end():]
+                        remainder = self.assign_attrs(elem, m.group(1))
+                        elem.tail = elem.tail[m.end():] + remainder
+
+    def assign_attrs(self, elem: Element, attrs_string: str, *, strict: bool = False) -> str:
+        """ Assign `attrs` to element.
+
+        If the `attrs_string` has an extra closing curly brace, the remaining text is returned.
+
+        The `strict` argument controls whether to still assign `attrs` if there is a remaining `}`.
+        """
+        attrs, remainder = get_attrs_and_remainder(attrs_string)
+        if strict and remainder:
+            return remainder
 
-    def assign_attrs(self, elem: Element, attrs: str) -> None:
-        """ Assign `attrs` to element. """
-        for k, v in get_attrs(attrs):
+        for k, v in attrs:
             if k == '.':
                 # add to class
                 cls = elem.get('class')
@@ -159,11 +181,13 @@ def assign_attrs(self, elem: Element, attrs: str) -> None:
             else:
                 # assign attribute `k` with `v`
                 elem.set(self.sanitize_name(k), v)
+        # The text that we initially over-matched will be put back.
+        return remainder
 
     def sanitize_name(self, name: str) -> str:
         """
-        Sanitize name as 'an XML Name, minus the ":"'.
-        See https://www.w3.org/TR/REC-xml-names/#NT-NCName
+        Sanitize name as 'an XML Name, minus the `:`.'
+        See <https://www.w3.org/TR/REC-xml-names/#NT-NCName>.
         """
         return self.NAME_RE.sub('_', name)
 

markdown/extensions/fenced_code.py

@@ -25,7 +25,7 @@
 from . import Extension
 from ..preprocessors import Preprocessor
 from .codehilite import CodeHilite, CodeHiliteExtension, parse_hl_lines
-from .attr_list import get_attrs, AttrListExtension
+from .attr_list import get_attrs_and_remainder, AttrListExtension
 from ..util import parseBoolValue
 from ..serializers import _escape_attrib_html
 import re
@@ -56,7 +56,7 @@ class FencedBlockPreprocessor(Preprocessor):
     FENCED_BLOCK_RE = re.compile(
         dedent(r'''
             (?P<fence>^(?:~{3,}|`{3,}))[ ]*                          # opening fence
-            ((\{(?P<attrs>[^\}\n]*)\})|                              # (optional {attrs} or
+            ((\{(?P<attrs>[^\n]*)\})|                                # (optional {attrs} or
             (\.?(?P<lang>[\w#.+-]*)[ ]*)?                            # optional (.)lang
             (hl_lines=(?P<quot>"|')(?P<hl_lines>.*?)(?P=quot)[ ]*)?) # optional hl_lines)
             \n                                                       # newline (end of opening fence)
@@ -94,12 +94,17 @@ def run(self, lines: list[str]) -> list[str]:
             self.checked_for_deps = True
 
         text = "\n".join(lines)
+        index = 0
         while 1:
-            m = self.FENCED_BLOCK_RE.search(text)
+            m = self.FENCED_BLOCK_RE.search(text, index)
             if m:
                 lang, id, classes, config = None, '', [], {}
                 if m.group('attrs'):
-                    id, classes, config = self.handle_attrs(get_attrs(m.group('attrs')))
+                    attrs, remainder = get_attrs_and_remainder(m.group('attrs'))
+                    if remainder:  # Does not have correctly matching curly braces, so the syntax is invalid.
+                        index = m.end('attrs')  # Explicitly skip over this, to prevent an infinite loop.
+                        continue
+                    id, classes, config = self.handle_attrs(attrs)
                     if len(classes):
                         lang = classes.pop(0)
                 else:
@@ -151,6 +156,8 @@ def run(self, lines: list[str]) -> list[str]:
 
                 placeholder = self.md.htmlStash.store(code)
                 text = f'{text[:m.start()]}\n{placeholder}\n{text[m.end():]}'
+                # Continue from after the replaced text in the next iteration.
+                index = m.start() + 1 + len(placeholder)
             else:
                 break
         return text.split("\n")

tests/test_syntax/extensions/test_attr_list.py

@@ -23,16 +23,53 @@
 
 
 class TestAttrList(TestCase):
-
     maxDiff = None
+    default_kwargs = {'extensions': ['attr_list']}
 
     # TODO: Move the rest of the `attr_list` tests here.
 
-    def test_empty_list(self):
+    def test_empty_attr_list(self):
         self.assertMarkdownRenders(
             '*foo*{ }',
-            '<p><em>foo</em>{ }</p>',
-            extensions=['attr_list']
+            '<p><em>foo</em>{ }</p>'
+        )
+
+    def test_curly_after_inline(self):
+        self.assertMarkdownRenders(
+            '*inline*{.a} } *text*{.a }}',
+            '<p><em class="a">inline</em> } <em class="a">text</em>}</p>'
+        )
+
+    def test_extra_eq_gets_ignored_inside_curly_inline(self):
+        # Undesired behavior but kept for historic compatibility.
+        self.assertMarkdownRenders(
+            '*inline*{data-test="x" =a} *text*',
+            '<p><em data-test="x">inline</em> <em>text</em></p>'
+        )
+
+    def test_curly_after_block(self):
+        self.assertMarkdownRenders(
+            '# Heading {.a} }',
+            '<h1>Heading {.a} }</h1>'
+        )
+
+    def test_curly_in_single_quote(self):
+        self.assertMarkdownRenders(
+            "# Heading {data-test='{}'}",
+            '<h1 data-test="{}">Heading</h1>'
+        )
+
+    def test_curly_in_double_quote(self):
+        self.assertMarkdownRenders(
+            '# Heading {data-test="{}"}',
+            '<h1 data-test="{}">Heading</h1>'
+        )
+
+    def test_unclosed_quote_ignored(self):
+        # Undesired behavior but kept for historic compatibility.
+        self.assertMarkdownRenders(
+            '# Heading {foo="bar}',
+            '<h1 foo="&quot;bar">Heading</h1>'
         )
 
     def test_table_td(self):

tests/test_syntax/extensions/test_fenced_code.py

@@ -394,6 +394,48 @@ def testFencedCodeEscapedAttrs(self):
             extensions=['fenced_code', 'attr_list']
         )
 
+    def testFencedCodeCurlyInAttrs(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                '''
+                ``` { data-test="{}" }
+                # Some python code
+                ```
+                '''
+            ),
+            self.dedent(
+                '''
+                <pre><code data-test="{}"># Some python code
+                </code></pre>
+                '''
+            ),
+            extensions=['fenced_code', 'attr_list']
+        )
+
+    def testFencedCodeMismatchedCurlyInAttrs(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                '''
+                ``` { data-test="{}" } }
+                # Some python code
+                ```
+                ```
+                test
+                ```
+                '''
+            ),
+            self.dedent(
+                '''
+                <p>``` { data-test="{}" } }</p>
+                <h1>Some python code</h1>
+                <pre><code></code></pre>
+                <p>test
+                ```</p>
+                '''
+            ),
+            extensions=['fenced_code', 'attr_list']
+        )
+
 
 class TestFencedCodeWithCodehilite(TestCase):