From 386fe2dfe9ba8c23d89e714c54a80f2e23ae3cf3 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 4 Jan 2023 18:14:57 +0100 Subject: [PATCH 01/20] =?UTF-8?q?=E2=9C=A8=20NEW:=20Add=20inventory=20read?= =?UTF-8?q?er=20and=20CLI=20(#656)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracted from sphinx for use independently. --- myst_parser/_compat.py | 10 +- myst_parser/inventory.py | 381 ++++++++++++++++++ pyproject.toml | 1 + tests/static/objects_v1.inv | 5 + tests/static/objects_v2.inv | Bin 0 -> 236 bytes tests/{test_cli.py => test_anchors.py} | 3 +- tests/test_inventory.py | 50 +++ tests/test_inventory/test_inv_cli_v1.yaml | 12 + .../test_inv_cli_v2_options0_.yaml | 24 ++ .../test_inv_cli_v2_options1_.yaml | 24 ++ .../test_inv_cli_v2_options2_.yaml | 8 + .../test_inv_cli_v2_options3_.yaml | 8 + tests/test_inventory/test_inv_filter.yml | 8 + .../test_inv_filter_fnmatch.yml | 32 ++ 14 files changed, 562 insertions(+), 4 deletions(-) create mode 100644 myst_parser/inventory.py create mode 100644 tests/static/objects_v1.inv create mode 100644 tests/static/objects_v2.inv rename tests/{test_cli.py => test_anchors.py} (93%) create mode 100644 tests/test_inventory.py create mode 100644 tests/test_inventory/test_inv_cli_v1.yaml create mode 100644 tests/test_inventory/test_inv_cli_v2_options0_.yaml create mode 100644 tests/test_inventory/test_inv_cli_v2_options1_.yaml create mode 100644 tests/test_inventory/test_inv_cli_v2_options2_.yaml create mode 100644 tests/test_inventory/test_inv_cli_v2_options3_.yaml create mode 100644 tests/test_inventory/test_inv_filter.yml create mode 100644 tests/test_inventory/test_inv_filter_fnmatch.yml diff --git a/myst_parser/_compat.py b/myst_parser/_compat.py index da6a8d76..8cd9128f 100644 --- a/myst_parser/_compat.py +++ b/myst_parser/_compat.py @@ -5,9 +5,15 @@ from docutils.nodes import Element if sys.version_info >= (3, 8): - from typing import Literal, Protocol, get_args, get_origin # noqa: F401 + from typing import Literal, Protocol, TypedDict, get_args, get_origin # noqa: F401 else: - from typing_extensions import Literal, Protocol, get_args, get_origin # noqa: F401 + from typing_extensions import ( # noqa: F401 + Literal, + Protocol, + TypedDict, + get_args, + get_origin, + ) def findall(node: Element) -> Callable[..., Iterable[Element]]: diff --git a/myst_parser/inventory.py b/myst_parser/inventory.py new file mode 100644 index 00000000..80082199 --- /dev/null +++ b/myst_parser/inventory.py @@ -0,0 +1,381 @@ +"""Logic for dealing with sphinx style inventories (e.g. `objects.inv`). + +These contain mappings of reference names to ids, scoped by domain and object type. + +This is adapted from the Sphinx inventory.py module. +We replicate it here, so that it can be used without Sphinx. +""" +from __future__ import annotations + +import argparse +import json +import re +import zlib +from dataclasses import asdict, dataclass +from fnmatch import fnmatchcase +from typing import IO, TYPE_CHECKING, Iterator +from urllib.request import urlopen + +import yaml + +from ._compat import TypedDict + +if TYPE_CHECKING: + from sphinx.util.typing import Inventory + + +class InventoryItemType(TypedDict): + """A single inventory item.""" + + loc: str + """The relative location of the item.""" + text: str | None + """Implicit text to show for the item.""" + + +class InventoryType(TypedDict): + """Inventory data.""" + + name: str + """The name of the project.""" + version: str + """The version of the project.""" + objects: dict[str, dict[str, dict[str, InventoryItemType]]] + """Mapping of domain -> object type -> name -> item.""" + + +def from_sphinx(inv: Inventory) -> InventoryType: + """Convert a Sphinx inventory to one that is JSON compliant.""" + project = "" + version = "" + objs: dict[str, dict[str, dict[str, InventoryItemType]]] = {} + for domain_obj_name, data in inv.items(): + if ":" not in domain_obj_name: + continue + + domain_name, obj_type = domain_obj_name.split(":", 1) + objs.setdefault(domain_name, {}).setdefault(obj_type, {}) + for refname, refdata in data.items(): + project, version, uri, text = refdata + objs[domain_name][obj_type][refname] = { + "loc": uri, + "text": None if (not text or text == "-") else text, + } + + return { + "name": project, + "version": version, + "objects": objs, + } + + +def to_sphinx(inv: InventoryType) -> Inventory: + """Convert a JSON compliant inventory to one that is Sphinx compliant.""" + objs: Inventory = {} + for domain_name, obj_types in inv["objects"].items(): + for obj_type, refs in obj_types.items(): + for refname, refdata in refs.items(): + objs.setdefault(f"{domain_name}:{obj_type}", {})[refname] = ( + inv["name"], + inv["version"], + refdata["loc"], + refdata["text"] or "-", + ) + return objs + + +def load(stream: IO) -> InventoryType: + """Load inventory data from a stream.""" + reader = InventoryFileReader(stream) + line = reader.readline().rstrip() + if line == "# Sphinx inventory version 1": + return _load_v1(reader) + elif line == "# Sphinx inventory version 2": + return _load_v2(reader) + else: + raise ValueError("invalid inventory header: %s" % line) + + +def _load_v1(stream: InventoryFileReader) -> InventoryType: + """Load inventory data (format v1) from a stream.""" + projname = stream.readline().rstrip()[11:] + version = stream.readline().rstrip()[11:] + invdata: InventoryType = { + "name": projname, + "version": version, + "objects": {}, + } + for line in stream.readlines(): + name, objtype, location = line.rstrip().split(None, 2) + # version 1 did not add anchors to the location + domain = "py" + if objtype == "mod": + objtype = "module" + location += "#module-" + name + else: + location += "#" + name + invdata["objects"].setdefault(domain, {}).setdefault(objtype, {}) + invdata["objects"][domain][objtype][name] = {"loc": location, "text": None} + + return invdata + + +def _load_v2(stream: InventoryFileReader) -> InventoryType: + """Load inventory data (format v2) from a stream.""" + projname = stream.readline().rstrip()[11:] + version = stream.readline().rstrip()[11:] + invdata: InventoryType = { + "name": projname, + "version": version, + "objects": {}, + } + line = stream.readline() + if "zlib" not in line: + raise ValueError("invalid inventory header (not compressed): %s" % line) + + for line in stream.read_compressed_lines(): + # be careful to handle names with embedded spaces correctly + m = re.match(r"(?x)(.+?)\s+(\S+)\s+(-?\d+)\s+?(\S*)\s+(.*)", line.rstrip()) + if not m: + continue + name: str + type: str + name, type, _, location, text = m.groups() + if ":" not in type: + # wrong type value. type should be in the form of "{domain}:{objtype}" + # + # Note: To avoid the regex DoS, this is implemented in python (refs: #8175) + continue + if ( + type == "py:module" + and type in invdata["objects"] + and name in invdata["objects"][type] + ): + # due to a bug in 1.1 and below, + # two inventory entries are created + # for Python modules, and the first + # one is correct + continue + if location.endswith("$"): + location = location[:-1] + name + domain, objtype = type.split(":", 1) + invdata["objects"].setdefault(domain, {}).setdefault(objtype, {}) + if not text or text == "-": + text = None + invdata["objects"][domain][objtype][name] = {"loc": location, "text": text} + return invdata + + +_BUFSIZE = 16 * 1024 + + +class InventoryFileReader: + """A file reader for an inventory file. + + This reader supports mixture of texts and compressed texts. + """ + + def __init__(self, stream: IO) -> None: + self.stream = stream + self.buffer = b"" + self.eof = False + + def read_buffer(self) -> None: + chunk = self.stream.read(_BUFSIZE) + if chunk == b"": + self.eof = True + self.buffer += chunk + + def readline(self) -> str: + pos = self.buffer.find(b"\n") + if pos != -1: + line = self.buffer[:pos].decode() + self.buffer = self.buffer[pos + 1 :] + elif self.eof: + line = self.buffer.decode() + self.buffer = b"" + else: + self.read_buffer() + line = self.readline() + + return line + + def readlines(self) -> Iterator[str]: + while not self.eof: + line = self.readline() + if line: + yield line + + def read_compressed_chunks(self) -> Iterator[bytes]: + decompressor = zlib.decompressobj() + while not self.eof: + self.read_buffer() + yield decompressor.decompress(self.buffer) + self.buffer = b"" + yield decompressor.flush() + + def read_compressed_lines(self) -> Iterator[str]: + buf = b"" + for chunk in self.read_compressed_chunks(): + buf += chunk + pos = buf.find(b"\n") + while pos != -1: + yield buf[:pos].decode() + buf = buf[pos + 1 :] + pos = buf.find(b"\n") + + +@dataclass +class InvMatch: + """A match from an inventory.""" + + inv: str + domain: str + otype: str + name: str + proj: str + version: str + uri: str + text: str + + def asdict(self) -> dict[str, str]: + return asdict(self) + + +def filter_inventories( + inventories: dict[str, Inventory], + ref_target: str, + *, + ref_inv: None | str = None, + ref_domain: None | str = None, + ref_otype: None | str = None, + fnmatch_target=False, +) -> Iterator[InvMatch]: + """Resolve a cross-reference in the loaded sphinx inventories. + + :param inventories: Mapping of inventory name to inventory data + :param ref_target: The target to search for + :param ref_inv: The name of the sphinx inventory to search, if None then + all inventories will be searched + :param ref_domain: The name of the domain to search, if None then all domains + will be searched + :param ref_otype: The type of object to search for, if None then all types will be searched + :param fnmatch_target: Whether to match ref_target using fnmatchcase + + :yields: matching results + """ + for inv_name, inv_data in inventories.items(): + + if ref_inv is not None and ref_inv != inv_name: + continue + + for domain_obj_name, data in inv_data.items(): + + if ":" not in domain_obj_name: + continue + + domain_name, obj_type = domain_obj_name.split(":", 1) + + if ref_domain is not None and ref_domain != domain_name: + continue + + if ref_otype is not None and ref_otype != obj_type: + continue + + if not fnmatch_target and ref_target in data: + yield ( + InvMatch( + inv_name, domain_name, obj_type, ref_target, *data[ref_target] + ) + ) + elif fnmatch_target: + for target in data: + if fnmatchcase(target, ref_target): + yield ( + InvMatch( + inv_name, domain_name, obj_type, target, *data[target] + ) + ) + + +def inventory_cli(inputs: None | list[str] = None): + """Command line interface for fetching and parsing an inventory.""" + parser = argparse.ArgumentParser(description="Parse an inventory file.") + parser.add_argument("uri", metavar="[URL|PATH]", help="URI of the inventory file") + parser.add_argument( + "-d", + "--domain", + metavar="DOMAIN", + help="Filter the inventory by domain pattern", + ) + parser.add_argument( + "-o", + "--object-type", + metavar="TYPE", + help="Filter the inventory by object type pattern", + ) + parser.add_argument( + "-n", + "--name", + metavar="NAME", + help="Filter the inventory by reference name pattern", + ) + parser.add_argument( + "-f", + "--format", + choices=["yaml", "json"], + default="yaml", + help="Output format (default: yaml)", + ) + args = parser.parse_args(inputs) + + if args.uri.startswith("http"): + try: + with urlopen(args.uri) as stream: + invdata = load(stream) + except Exception: + with urlopen(args.uri + "/objects.inv") as stream: + invdata = load(stream) + else: + with open(args.uri, "rb") as stream: + invdata = load(stream) + + # filter the inventory + if args.domain: + invdata["objects"] = { + d: invdata["objects"][d] + for d in invdata["objects"] + if fnmatchcase(d, args.domain) + } + if args.object_type: + for domain in list(invdata["objects"]): + invdata["objects"][domain] = { + t: invdata["objects"][domain][t] + for t in invdata["objects"][domain] + if fnmatchcase(t, args.object_type) + } + if args.name: + for domain in invdata["objects"]: + for otype in list(invdata["objects"][domain]): + invdata["objects"][domain][otype] = { + n: invdata["objects"][domain][otype][n] + for n in invdata["objects"][domain][otype] + if fnmatchcase(n, args.name) + } + + # clean up empty items + for domain in list(invdata["objects"]): + for otype in list(invdata["objects"][domain]): + if not invdata["objects"][domain][otype]: + del invdata["objects"][domain][otype] + if not invdata["objects"][domain]: + del invdata["objects"][domain] + + if args.format == "json": + print(json.dumps(invdata, indent=2, sort_keys=False)) + else: + print(yaml.dump(invdata, sort_keys=False)) + + +if __name__ == "__main__": + inventory_cli() diff --git a/pyproject.toml b/pyproject.toml index 5b4484fa..9bb68ab2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ testing = [ [project.scripts] myst-anchors = "myst_parser.cli:print_anchors" +myst-inv = "myst_parser.inventory:inventory_cli" myst-docutils-html = "myst_parser.parsers.docutils_:cli_html" myst-docutils-html5 = "myst_parser.parsers.docutils_:cli_html5" myst-docutils-latex = "myst_parser.parsers.docutils_:cli_latex" diff --git a/tests/static/objects_v1.inv b/tests/static/objects_v1.inv new file mode 100644 index 00000000..66947723 --- /dev/null +++ b/tests/static/objects_v1.inv @@ -0,0 +1,5 @@ +# Sphinx inventory version 1 +# Project: foo +# Version: 1.0 +module mod foo.html +module.cls class foo.html diff --git a/tests/static/objects_v2.inv b/tests/static/objects_v2.inv new file mode 100644 index 0000000000000000000000000000000000000000..f620d7639f7b62d36171345b5a6eacdc79d694b4 GIT binary patch literal 236 zcmY#Z2rkIT%&Sny%qvUHE6FdaR47X=D$dN$Q!wIERtPA{&q_@$u~G=AEXl~v1B!$} zWUUl{?2wF9g`(8l#LT>u)FOraG=-9k%wmPK%$!sOAf23_TTql*T%4MsP+FXsm#$Ei zlbNK)RdLJP|Lo~A-kxg%H1s?-p7QkZIvaSwG{mF5>s9KMC(kr0nr6gsq-y>=so?6N z8 gF1lPzOf`LhR&$5r8Q!)N>?+Ha7cnx--x#+D0Popd3IG5A literal 0 HcmV?d00001 diff --git a/tests/test_cli.py b/tests/test_anchors.py similarity index 93% rename from tests/test_cli.py rename to tests/test_anchors.py index 4725f930..8092f183 100644 --- a/tests/test_cli.py +++ b/tests/test_anchors.py @@ -1,11 +1,10 @@ +from io import StringIO from unittest import mock from myst_parser.cli import print_anchors def test_print_anchors(): - from io import StringIO - in_stream = StringIO("# a\n\n## b\n\ntext") out_stream = StringIO() with mock.patch("sys.stdin", in_stream): diff --git a/tests/test_inventory.py b/tests/test_inventory.py new file mode 100644 index 00000000..f6d1b3d2 --- /dev/null +++ b/tests/test_inventory.py @@ -0,0 +1,50 @@ +"""Test reading of inventory files.""" +from pathlib import Path + +import pytest + +from myst_parser.inventory import ( + filter_inventories, + from_sphinx, + inventory_cli, + load, + to_sphinx, +) + +STATIC = Path(__file__).parent.absolute() / "static" + + +def test_convert_roundtrip(): + with (STATIC / "objects_v2.inv").open("rb") as f: + inv = load(f) + assert inv == from_sphinx(to_sphinx(inv)) + + +def test_inv_filter(data_regression): + with (STATIC / "objects_v2.inv").open("rb") as f: + inv = to_sphinx(load(f)) + output = [m.asdict() for m in filter_inventories({"inv": inv}, "index")] + data_regression.check(output) + + +def test_inv_filter_fnmatch(data_regression): + with (STATIC / "objects_v2.inv").open("rb") as f: + inv = to_sphinx(load(f)) + output = [ + m.asdict() + for m in filter_inventories({"inv": inv}, "*index", fnmatch_target=True) + ] + data_regression.check(output) + + +@pytest.mark.parametrize("options", [(), ("-d", "std"), ("-o", "doc"), ("-n", "ref")]) +def test_inv_cli_v2(options, capsys, file_regression): + inventory_cli([str(STATIC / "objects_v2.inv"), "-f", "yaml", *options]) + text = capsys.readouterr().out.strip() + "\n" + file_regression.check(text, extension=".yaml") + + +def test_inv_cli_v1(capsys, file_regression): + inventory_cli([str(STATIC / "objects_v1.inv"), "-f", "yaml"]) + text = capsys.readouterr().out.strip() + "\n" + file_regression.check(text, extension=".yaml") diff --git a/tests/test_inventory/test_inv_cli_v1.yaml b/tests/test_inventory/test_inv_cli_v1.yaml new file mode 100644 index 00000000..1eb78b85 --- /dev/null +++ b/tests/test_inventory/test_inv_cli_v1.yaml @@ -0,0 +1,12 @@ +name: foo +version: '1.0' +objects: + py: + module: + module: + loc: foo.html#module-module + text: null + class: + module.cls: + loc: foo.html#module.cls + text: null diff --git a/tests/test_inventory/test_inv_cli_v2_options0_.yaml b/tests/test_inventory/test_inv_cli_v2_options0_.yaml new file mode 100644 index 00000000..d51b779f --- /dev/null +++ b/tests/test_inventory/test_inv_cli_v2_options0_.yaml @@ -0,0 +1,24 @@ +name: Python +version: '' +objects: + std: + label: + genindex: + loc: genindex.html + text: Index + modindex: + loc: py-modindex.html + text: Module Index + py-modindex: + loc: py-modindex.html + text: Python Module Index + ref: + loc: index.html#ref + text: Title + search: + loc: search.html + text: Search Page + doc: + index: + loc: index.html + text: Title diff --git a/tests/test_inventory/test_inv_cli_v2_options1_.yaml b/tests/test_inventory/test_inv_cli_v2_options1_.yaml new file mode 100644 index 00000000..d51b779f --- /dev/null +++ b/tests/test_inventory/test_inv_cli_v2_options1_.yaml @@ -0,0 +1,24 @@ +name: Python +version: '' +objects: + std: + label: + genindex: + loc: genindex.html + text: Index + modindex: + loc: py-modindex.html + text: Module Index + py-modindex: + loc: py-modindex.html + text: Python Module Index + ref: + loc: index.html#ref + text: Title + search: + loc: search.html + text: Search Page + doc: + index: + loc: index.html + text: Title diff --git a/tests/test_inventory/test_inv_cli_v2_options2_.yaml b/tests/test_inventory/test_inv_cli_v2_options2_.yaml new file mode 100644 index 00000000..9ea4200f --- /dev/null +++ b/tests/test_inventory/test_inv_cli_v2_options2_.yaml @@ -0,0 +1,8 @@ +name: Python +version: '' +objects: + std: + doc: + index: + loc: index.html + text: Title diff --git a/tests/test_inventory/test_inv_cli_v2_options3_.yaml b/tests/test_inventory/test_inv_cli_v2_options3_.yaml new file mode 100644 index 00000000..e64e40ee --- /dev/null +++ b/tests/test_inventory/test_inv_cli_v2_options3_.yaml @@ -0,0 +1,8 @@ +name: Python +version: '' +objects: + std: + label: + ref: + loc: index.html#ref + text: Title diff --git a/tests/test_inventory/test_inv_filter.yml b/tests/test_inventory/test_inv_filter.yml new file mode 100644 index 00000000..8ac5d5f0 --- /dev/null +++ b/tests/test_inventory/test_inv_filter.yml @@ -0,0 +1,8 @@ +- domain: std + inv: inv + name: index + otype: doc + proj: Python + text: Title + uri: index.html + version: '' diff --git a/tests/test_inventory/test_inv_filter_fnmatch.yml b/tests/test_inventory/test_inv_filter_fnmatch.yml new file mode 100644 index 00000000..aa24f602 --- /dev/null +++ b/tests/test_inventory/test_inv_filter_fnmatch.yml @@ -0,0 +1,32 @@ +- domain: std + inv: inv + name: genindex + otype: label + proj: Python + text: Index + uri: genindex.html + version: '' +- domain: std + inv: inv + name: modindex + otype: label + proj: Python + text: Module Index + uri: py-modindex.html + version: '' +- domain: std + inv: inv + name: py-modindex + otype: label + proj: Python + text: Python Module Index + uri: py-modindex.html + version: '' +- domain: std + inv: inv + name: index + otype: doc + proj: Python + text: Title + uri: index.html + version: '' From 8ea5398a6f8042f70e91233f20dc5778e0008fe8 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 4 Jan 2023 19:54:57 +0100 Subject: [PATCH 02/20] =?UTF-8?q?=F0=9F=A7=AA=20TESTS:=20Fix=20for=20sphin?= =?UTF-8?q?x=205.3=20(#663)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tests.yml | 2 +- pyproject.toml | 1 - tests/test_renderers/test_fixtures_sphinx.py | 7 +++++ .../test_renderers/test_include_directive.py | 5 ++++ tests/test_sphinx/conftest.py | 3 ++ tests/test_sphinx/test_sphinx_builds.py | 17 +++++++++-- .../test_sphinx_builds/test_includes.xml | 30 +++++++++---------- 7 files changed, 45 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c05356e6..208652db 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,7 +54,7 @@ jobs: pytest --cov=myst_parser --cov-report=xml --cov-report=term-missing coverage xml - name: Upload to Codecov - if: github.repository == 'executablebooks/MyST-Parser' && matrix.python-version == 3.8 + if: github.repository == 'executablebooks/MyST-Parser' && matrix.python-version == 3.8 && matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v1 with: name: myst-parser-pytests diff --git a/pyproject.toml b/pyproject.toml index 9bb68ab2..82cfb5e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,6 @@ testing = [ "pytest-regressions", "pytest-param-files~=0.3.4", "sphinx-pytest", - "sphinx<5.2", # TODO 5.2 changes the attributes of desc/desc_signature nodes ] [project.scripts] diff --git a/tests/test_renderers/test_fixtures_sphinx.py b/tests/test_renderers/test_fixtures_sphinx.py index b8cf5497..735ba705 100644 --- a/tests/test_renderers/test_fixtures_sphinx.py +++ b/tests/test_renderers/test_fixtures_sphinx.py @@ -53,6 +53,13 @@ def test_sphinx_directives(file_params, sphinx_doctree_no_tr: CreateDoctree): # see https://github.com/executablebooks/MyST-Parser/issues/522 if sys.maxsize == 2147483647: pformat = pformat.replace('"2147483647"', '"9223372036854775807"') + # changed in sphinx 5.3 for desc node + pformat = pformat.replace('nocontentsentry="False" ', "") + pformat = pformat.replace('noindexentry="False" ', "") + # changed in sphinx 5.3 for desc_signature node + pformat = pformat.replace('_toc_name="" _toc_parts="()" ', "") + pformat = pformat.replace('_toc_name=".. a::" _toc_parts="(\'a\',)" ', "") + pformat = pformat.replace('fullname="a" ', "") file_params.assert_expected(pformat, rstrip_lines=True) diff --git a/tests/test_renderers/test_include_directive.py b/tests/test_renderers/test_include_directive.py index f02b246f..4a2f2f6c 100644 --- a/tests/test_renderers/test_include_directive.py +++ b/tests/test_renderers/test_include_directive.py @@ -24,6 +24,11 @@ def test_render(file_params, tmp_path, monkeypatch): ) doctree["source"] = "tmpdir/test.md" + if file_params.title.startswith("Include code:"): + # from sphinx 5.3 whitespace nodes are now present + for node in doctree.traverse(): + if node.tagname == "inline" and node["classes"] == ["whitespace"]: + node.parent.remove(node) output = doctree.pformat().replace(str(tmp_path) + os.sep, "tmpdir" + "/").rstrip() file_params.assert_expected(output, rstrip=True) diff --git a/tests/test_sphinx/conftest.py b/tests/test_sphinx/conftest.py index 4165a318..6ce0cb91 100644 --- a/tests/test_sphinx/conftest.py +++ b/tests/test_sphinx/conftest.py @@ -101,6 +101,7 @@ def read( resolve=False, regress=False, replace=None, + rstrip_lines=False, regress_ext=".xml", ): if resolve: @@ -120,6 +121,8 @@ def read( text = doctree.pformat() # type: str for find, rep in (replace or {}).items(): text = text.replace(find, rep) + if rstrip_lines: + text = "\n".join([li.rstrip() for li in text.splitlines()]) file_regression.check(text, extension=extension) return doctree diff --git a/tests/test_sphinx/test_sphinx_builds.py b/tests/test_sphinx/test_sphinx_builds.py index beba4845..0d45a424 100644 --- a/tests/test_sphinx/test_sphinx_builds.py +++ b/tests/test_sphinx/test_sphinx_builds.py @@ -248,11 +248,14 @@ def test_includes( app, docname="index", regress=True, + rstrip_lines=True, # fix for Windows CI replace={ r"subfolder\example2.jpg": "subfolder/example2.jpg", r"subfolder\\example2.jpg": "subfolder/example2.jpg", r"subfolder\\\\example2.jpg": "subfolder/example2.jpg", + # in sphinx 5.3 whitespace nodes were added + ' \n ': "", }, ) finally: @@ -265,6 +268,9 @@ def test_includes( r"'subfolder\\example2'": "'subfolder/example2'", r'uri="subfolder\\example2"': 'uri="subfolder/example2"', "_images/example21.jpg": "_images/example2.jpg", + # in sphinx 5.3 whitespace nodes were added + '': "", + '\n': "\n", }, ) @@ -530,13 +536,18 @@ def test_fieldlist_extension( app, docname="index", regress=True, - # changed in: - # https://www.sphinx-doc.org/en/master/changes.html#release-4-4-0-released-jan-17-2022 replace={ + # changed in: + # https://www.sphinx-doc.org/en/master/changes.html#release-4-4-0-released-jan-17-2022 ( '' - ): "" + ): "", + # changed in sphinx 5.3, for `desc` node + 'nocontentsentry="False" ': "", + 'noindexentry="False" ': "", + # changed in sphinx 5.3, for `desc_signature` node + '_toc_name="send_message()" _toc_parts="(\'send_message\',)" ': "", }, ) finally: diff --git a/tests/test_sphinx/test_sphinx_builds/test_includes.xml b/tests/test_sphinx/test_sphinx_builds/test_includes.xml index 1e8779c5..66024070 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_includes.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_includes.xml @@ -7,14 +7,14 @@ A Sub-Heading in Include <paragraph> - Some text with + Some text with <emphasis> syntax <section ids="a-sub-heading-in-nested-include" names="a\ sub-heading\ in\ nested\ include"> <title> A Sub-Heading in Nested Include <paragraph> - Some other text with + Some other text with <strong> syntax <paragraph> @@ -24,7 +24,7 @@ <caption> Caption <paragraph> - This absolute path will refer to the project root (where the + This absolute path will refer to the project root (where the <literal> conf.py is): @@ -47,7 +47,7 @@ <literal_block classes="code python" source="include_code.py" xml:space="preserve"> <inline classes="keyword"> def - + <inline classes="name function"> a_func <inline classes="punctuation"> @@ -56,8 +56,8 @@ param <inline classes="punctuation"> ): - - + + <inline classes="name builtin"> print <inline classes="punctuation"> @@ -68,10 +68,10 @@ ) <literal_block classes="code python" source="include_code.py" xml:space="preserve"> <inline classes="ln"> - 0 + 0 <inline classes="keyword"> def - + <inline classes="name function"> a_func <inline classes="punctuation"> @@ -80,10 +80,10 @@ param <inline classes="punctuation"> ): - + <inline classes="ln"> - 1 - + 1 + <inline classes="name builtin"> print <inline classes="punctuation"> @@ -94,20 +94,20 @@ ) <literal_block source="include_literal.txt" xml:space="preserve"> This should be *literal* - + Lots of lines so we can select some <literal_block ids="literal-ref" names="literal_ref" source="include_literal.txt" xml:space="preserve"> <inline classes="ln"> - 0 + 0 Lots <inline classes="ln"> - 1 + 1 of <section ids="a-sub-sub-heading" names="a\ sub-sub-heading"> <title> A Sub-sub-Heading <paragraph> - some more text + some more text \ No newline at end of file From 9a4de682c374c3acc77cae4a1544f4169b1552e8 Mon Sep 17 00:00:00 2001 From: Nicolas Peugnet <n.peugnet@free.fr> Date: Thu, 5 Jan 2023 00:18:50 +0100 Subject: [PATCH 03/20] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Remove=20unnecessar?= =?UTF-8?q?y=20assert=20(#659)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Chris Sewell <chrisj_sewell@hotmail.com> Fixes https://github.com/executablebooks/MyST-Parser/issues/657 --- myst_parser/config/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py index a134ea7d..1223a286 100644 --- a/myst_parser/config/main.py +++ b/myst_parser/config/main.py @@ -401,7 +401,6 @@ def read_topmatter(text: Union[str, Iterator[str]]) -> Optional[Dict[str, Any]]: top_matter.append(line.rstrip() + "\n") try: metadata = yaml.safe_load("".join(top_matter)) - assert isinstance(metadata, dict) except (yaml.parser.ParserError, yaml.scanner.ScannerError) as err: raise TopmatterReadError("Malformed YAML") from err if not isinstance(metadata, dict): From 1b84a5bc1dee099302ffcaae74a6c8ef80bf616b Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Thu, 5 Jan 2023 00:22:44 +0100 Subject: [PATCH 04/20] =?UTF-8?q?=E2=9C=A8=20NEW:=20suppress=20warnings=20?= =?UTF-8?q?in=20docutils=20(#655)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit with `myst_suppress_warnings` option, which works the same as sphinx `suppress_warnings`. --- CHANGELOG.md | 2 +- docs/conf.py | 2 + docs/configuration.md | 23 ++++ docs/faq/index.md | 21 +--- docs/syntax/optional.md | 2 +- myst_parser/_docs.py | 32 +++++- myst_parser/config/main.py | 43 +++++--- myst_parser/mdit_to_docutils/base.py | 58 ++++------ myst_parser/mdit_to_docutils/html_to_nodes.py | 3 +- myst_parser/mdit_to_docutils/sphinx_.py | 55 +--------- myst_parser/parsers/docutils_.py | 9 +- myst_parser/parsers/sphinx_.py | 5 +- myst_parser/sphinx_ext/myst_refs.py | 5 +- myst_parser/warnings_.py | 101 ++++++++++++++++++ tests/test_renderers/fixtures/myst-config.txt | 24 +++-- .../fixtures/reporter_warnings.md | 2 +- 16 files changed, 241 insertions(+), 146 deletions(-) create mode 100644 myst_parser/warnings_.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fe72f45..37a50b77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -437,7 +437,7 @@ A warning (of type `myst.nested_header`) is now emitted when this occurs. - ✨ NEW: Add warning types `myst.subtype`: All parsing warnings are assigned a type/subtype, and also the messages are appended with them. These warning types can be suppressed with the sphinx `suppress_warnings` config option. - See [How-to suppress warnings](howto/warnings) for more information. + See [How-to suppress warnings](myst-warnings) for more information. ## 0.13.3 - 2021-01-20 diff --git a/docs/conf.py b/docs/conf.py index 199a32a4..7db210c5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -161,9 +161,11 @@ def setup(app: Sphinx): DirectiveDoc, DocutilsCliHelpDirective, MystConfigDirective, + MystWarningsDirective, ) app.add_css_file("custom.css") app.add_directive("myst-config", MystConfigDirective) app.add_directive("docutils-cli-help", DocutilsCliHelpDirective) app.add_directive("doc-directive", DirectiveDoc) + app.add_directive("myst-warnings", MystWarningsDirective) diff --git a/docs/configuration.md b/docs/configuration.md index a87ce0bd..286a612a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -104,3 +104,26 @@ substitution tasklist : Add check-boxes to the start of list items, [see here](syntax/tasklists) for details + +(howto/warnings)= +(myst-warnings)= +## Build Warnings + +Below lists the MyST specific warnings that may be emitted during the build process. These will be prepended to the end of the warning message, e.g. + +``` +WARNING: Non-consecutive header level increase; H1 to H3 [myst.header] +``` + +**In general, if your build logs any warnings, you should either fix them or [raise an Issue](https://github.com/executablebooks/MyST-Parser/issues/new/choose) if you think the warning is erroneous.** + +However, in some circumstances if you wish to suppress the warning you can use the [`suppress_warnings`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-suppress_warnings) configuration option, e.g. + +```python +suppress_warnings = ["myst.header"] +``` + +Or use `--myst-suppress-warnings="myst.header"` for the [docutils CLI](myst-docutils). + +```{myst-warnings} +``` diff --git a/docs/faq/index.md b/docs/faq/index.md index e4d45815..a62f861d 100644 --- a/docs/faq/index.md +++ b/docs/faq/index.md @@ -172,28 +172,9 @@ like so: {ref}`path/to/file_1:My Subtitle` ``` -(howto/warnings)= ### Suppress warnings -In general, if your build logs any warnings, you should either fix them or [raise an Issue](https://github.com/executablebooks/MyST-Parser/issues/new/choose) if you think the warning is erroneous. -However, in some circumstances if you wish to suppress the warning you can use the [`suppress_warnings`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-suppress_warnings) configuration option. -All myst-parser warnings are prepended by their type, e.g. to suppress: - -```md -# Title -### Subtitle -``` - -``` -WARNING: Non-consecutive header level increase; H1 to H3 [myst.header] -``` - -Add to your `conf.py`: - -```python -suppress_warnings = ["myst.header"] -``` - +Moved to [](myst-warnings) ### Sphinx-specific page front matter diff --git a/docs/syntax/optional.md b/docs/syntax/optional.md index c5ee44ed..b11ab08b 100644 --- a/docs/syntax/optional.md +++ b/docs/syntax/optional.md @@ -88,7 +88,7 @@ For example, `~~strikethrough with *emphasis*~~` renders as: ~~strikethrough wit :::{warning} This extension is currently only supported for HTML output, and you will need to suppress the `myst.strikethrough` warning -(see [](howto/warnings)) +(see [](myst-warnings)) ::: (syntax/math)= diff --git a/myst_parser/_docs.py b/myst_parser/_docs.py index 6644bb38..cdef6c46 100644 --- a/myst_parser/_docs.py +++ b/myst_parser/_docs.py @@ -14,8 +14,9 @@ from ._compat import get_args, get_origin from .config.main import MdParserConfig from .parsers.docutils_ import Parser as DocutilsParser +from .warnings_ import MystWarnings -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class _ConfigBase(SphinxDirective): @@ -72,6 +73,9 @@ def run(self): count = 0 for name, value, field in config.as_triple(): + if field.metadata.get("deprecated"): + continue + # filter by sphinx options if "sphinx" in self.options and field.metadata.get("sphinx_exclude"): continue @@ -146,7 +150,7 @@ def run(self): name, self.state.memo.language, self.state.document ) if klass is None: - logger.warning(f"Directive {name} not found.", line=self.lineno) + LOGGER.warning(f"Directive {name} not found.", line=self.lineno) return [] content = " ".join(self.content) text = f"""\ @@ -196,3 +200,27 @@ def convert_opt(name, func): if func is other.int_or_nothing: return "integer" return "" + + +class MystWarningsDirective(SphinxDirective): + """Directive to print all known warnings.""" + + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = False + + def run(self): + """Run the directive.""" + from sphinx.pycode import ModuleAnalyzer + + analyzer = ModuleAnalyzer.for_module(MystWarnings.__module__) + qname = MystWarnings.__qualname__ + analyzer.analyze() + warning_names = [ + (e.value, analyzer.attr_docs[(qname, e.name)]) for e in MystWarnings + ] + text = [f"- `myst.{name}`: {' '.join(doc)}" for name, doc in warning_names] + node = nodes.Element() + self.state.nested_parse(text, 0, node) + return node.children diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py index 1223a286..118c3d08 100644 --- a/myst_parser/config/main.py +++ b/myst_parser/config/main.py @@ -13,6 +13,7 @@ cast, ) +from myst_parser.warnings_ import MystWarnings from .dc_validators import ( deep_iterable, deep_mapping, @@ -126,15 +127,6 @@ class MdParserConfig: }, ) - highlight_code_blocks: bool = dc.field( - default=True, - metadata={ - "validator": instance_of(bool), - "help": "Syntax highlight code blocks with pygments", - "docutils_only": True, - }, - ) - number_code_blocks: Sequence[str] = dc.field( default_factory=list, metadata={ @@ -282,6 +274,27 @@ class MdParserConfig: }, ) + # docutils only (replicating aspects of sphinx config) + + suppress_warnings: Sequence[str] = dc.field( + default_factory=list, + metadata={ + "validator": deep_iterable(instance_of(str), instance_of((list, tuple))), + "help": "A list of warning types to suppress warning messages", + "docutils_only": True, + "global_only": True, + }, + ) + + highlight_code_blocks: bool = dc.field( + default=True, + metadata={ + "validator": instance_of(bool), + "help": "Syntax highlight code blocks with pygments", + "docutils_only": True, + }, + ) + def __post_init__(self): validate_fields(self) @@ -311,7 +324,7 @@ def as_triple(self) -> Iterable[Tuple[str, Any, dc.Field]]: def merge_file_level( config: MdParserConfig, topmatter: Dict[str, Any], - warning: Callable[[str, str], None], + warning: Callable[[MystWarnings, str], None], ) -> MdParserConfig: """Merge the file-level topmatter with the global config. @@ -324,21 +337,21 @@ def merge_file_level( updates: Dict[str, Any] = {} myst = topmatter.get("myst", {}) if not isinstance(myst, dict): - warning("topmatter", f"'myst' key not a dict: {type(myst)}") + warning(MystWarnings.MD_TOPMATTER, f"'myst' key not a dict: {type(myst)}") else: updates = myst # allow html_meta and substitutions at top-level for back-compatibility if "html_meta" in topmatter: warning( - "topmatter", + MystWarnings.MD_TOPMATTER, "top-level 'html_meta' key is deprecated, " "place under 'myst' key instead", ) updates["html_meta"] = topmatter["html_meta"] if "substitutions" in topmatter: warning( - "topmatter", + MystWarnings.MD_TOPMATTER, "top-level 'substitutions' key is deprecated, " "place under 'myst' key instead", ) @@ -351,7 +364,7 @@ def merge_file_level( for name, value in updates.items(): if name not in fields: - warning("topmatter", f"Unknown field: {name}") + warning(MystWarnings.MD_TOPMATTER, f"Unknown field: {name}") continue old_value, field = fields[name] @@ -359,7 +372,7 @@ def merge_file_level( try: validate_field(new, field, value) except Exception as exc: - warning("topmatter", str(exc)) + warning(MystWarnings.MD_TOPMATTER, str(exc)) continue if field.metadata.get("merge_topmatter"): diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index cedd6c35..f21e6d11 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -43,6 +43,7 @@ MockStateMachine, ) from myst_parser.parsers.directives import DirectiveParsingError, parse_directive_text +from myst_parser.warnings_ import MystWarnings, create_warning from .html_to_nodes import html_to_nodes from .utils import is_external_url @@ -68,27 +69,6 @@ def token_line(token: SyntaxTreeNode, default: int | None = None) -> int: return token.map[0] # type: ignore[index] -def create_warning( - document: nodes.document, - message: str, - *, - line: int | None = None, - append_to: nodes.Element | None = None, - wtype: str = "myst", - subtype: str = "other", -) -> nodes.system_message | None: - """Generate a warning, logging if it is necessary. - - Note this is overridden in the ``SphinxRenderer``, - to handle suppressed warning types. - """ - kwargs = {"line": line} if line is not None else {} - msg_node = document.reporter.warning(f"{message} [{wtype}.{subtype}]", **kwargs) - if append_to is not None: - append_to.append(msg_node) - return msg_node - - class DocutilsRenderer(RendererProtocol): """A markdown-it-py renderer to populate (in-place) a `docutils.document` AST. @@ -156,24 +136,22 @@ def sphinx_env(self) -> BuildEnvironment | None: def create_warning( self, message: str, + subtype: MystWarnings, *, line: int | None = None, append_to: nodes.Element | None = None, - wtype: str = "myst", - subtype: str = "other", ) -> nodes.system_message | None: """Generate a warning, logging if it is necessary. - Note this is overridden in the ``SphinxRenderer``, - to handle suppressed warning types. + If the warning type is listed in the ``suppress_warnings`` configuration, + then ``None`` will be returned and no warning logged. """ return create_warning( self.document, message, + subtype, line=line, append_to=append_to, - wtype=wtype, - subtype=subtype, ) def _render_tokens(self, tokens: list[Token]) -> None: @@ -211,8 +189,8 @@ def _render_tokens(self, tokens: list[Token]) -> None: else: self.create_warning( f"No render method for: {child.type}", + MystWarnings.RENDER_METHOD, line=token_line(child, default=0), - subtype="render", append_to=self.current_node, ) @@ -251,8 +229,8 @@ def _render_finalise(self) -> None: for dup_ref in self.md_env.get("duplicate_refs", []): self.create_warning( f"Duplicate reference definition: {dup_ref['label']}", + MystWarnings.MD_DEF_DUPE, line=dup_ref["map"][0] + 1, - subtype="ref", append_to=self.document, ) @@ -272,14 +250,14 @@ def _render_finalise(self) -> None: if len(foot_ref_tokens) > 1: self.create_warning( f"Multiple footnote definitions found for label: '{footref}'", - subtype="footnote", + MystWarnings.MD_FOOTNOTE_DUPE, append_to=self.current_node, ) if len(foot_ref_tokens) < 1: self.create_warning( f"No footnote definitions found for label: '{footref}'", - subtype="footnote", + MystWarnings.MD_FOOTNOTE_MISSING, append_to=self.current_node, ) else: @@ -360,8 +338,8 @@ def render_children(self, token: SyntaxTreeNode) -> None: else: self.create_warning( f"No render method for: {child.type}", + MystWarnings.RENDER_METHOD, line=token_line(child, default=0), - subtype="render", append_to=self.current_node, ) @@ -402,8 +380,8 @@ def update_section_level_state(self, section: nodes.section, level: int) -> None msg = f"Document headings start at H{level}, not H1" self.create_warning( msg, + MystWarnings.MD_HEADING_NON_CONSECUTIVE, line=section.line, - subtype="header", append_to=self.current_node, ) @@ -638,8 +616,8 @@ def render_heading(self, token: SyntaxTreeNode) -> None: # this would break the document structure self.create_warning( "Disallowed nested header found, converting to rubric", + MystWarnings.MD_HEADING_NESTED, line=token_line(token, default=0), - subtype="nested_header", append_to=self.current_node, ) rubric = nodes.rubric(token.content, "") @@ -797,8 +775,8 @@ def render_image(self, token: SyntaxTreeNode) -> None: except ValueError: self.create_warning( f"Invalid width value for image: {token.attrs['width']!r}", + MystWarnings.INVALID_ATTRIBUTE, line=token_line(token, default=0), - subtype="image", append_to=self.current_node, ) else: @@ -809,8 +787,8 @@ def render_image(self, token: SyntaxTreeNode) -> None: except ValueError: self.create_warning( f"Invalid height value for image: {token.attrs['height']!r}", + MystWarnings.INVALID_ATTRIBUTE, line=token_line(token, default=0), - subtype="image", append_to=self.current_node, ) else: @@ -819,8 +797,8 @@ def render_image(self, token: SyntaxTreeNode) -> None: if token.attrs["align"] not in ("left", "center", "right"): self.create_warning( f"Invalid align value for image: {token.attrs['align']!r}", + MystWarnings.INVALID_ATTRIBUTE, line=token_line(token, default=0), - subtype="image", append_to=self.current_node, ) else: @@ -844,9 +822,9 @@ def render_front_matter(self, token: SyntaxTreeNode) -> None: except (yaml.parser.ParserError, yaml.scanner.ScannerError): self.create_warning( "Malformed YAML", + MystWarnings.MD_TOPMATTER, line=position, append_to=self.current_node, - subtype="topmatter", ) return else: @@ -855,9 +833,9 @@ def render_front_matter(self, token: SyntaxTreeNode) -> None: if not isinstance(data, dict): self.create_warning( f"YAML is not a dict: {type(data)}", + MystWarnings.MD_TOPMATTER, line=position, append_to=self.current_node, - subtype="topmatter", ) return @@ -1004,8 +982,8 @@ def render_s(self, token: SyntaxTreeNode) -> None: # TODO strikethrough not currently directly supported in docutils self.create_warning( "Strikethrough is currently only supported in HTML output", + MystWarnings.STRIKETHROUGH, line=token_line(token, 0), - subtype="strikethrough", append_to=self.current_node, ) self.current_node.append(nodes.raw("", "<s>", format="html")) diff --git a/myst_parser/mdit_to_docutils/html_to_nodes.py b/myst_parser/mdit_to_docutils/html_to_nodes.py index 2cc30667..539405d1 100644 --- a/myst_parser/mdit_to_docutils/html_to_nodes.py +++ b/myst_parser/mdit_to_docutils/html_to_nodes.py @@ -7,6 +7,7 @@ from docutils import nodes from myst_parser.parsers.parse_html import Data, tokenize_html +from myst_parser.warnings_ import MystWarnings if TYPE_CHECKING: from .base import DocutilsRenderer @@ -58,7 +59,7 @@ def html_to_nodes( root = tokenize_html(text).strip(inplace=True, recurse=False) except Exception: msg_node = renderer.create_warning( - "HTML could not be parsed", line=line_number, subtype="html" + "HTML could not be parsed", MystWarnings.HTML_PARSE, line=line_number ) return ([msg_node] if msg_node else []) + default_html( text, renderer.document["source"], line_number diff --git a/myst_parser/mdit_to_docutils/sphinx_.py b/myst_parser/mdit_to_docutils/sphinx_.py index 3c1bc237..b6c55726 100644 --- a/myst_parser/mdit_to_docutils/sphinx_.py +++ b/myst_parser/mdit_to_docutils/sphinx_.py @@ -17,39 +17,11 @@ from sphinx.util.nodes import clean_astext from myst_parser.mdit_to_docutils.base import DocutilsRenderer +from myst_parser.warnings_ import MystWarnings LOGGER = logging.getLogger(__name__) -def create_warning( - document: nodes.document, - message: str, - *, - line: int | None = None, - append_to: nodes.Element | None = None, - wtype: str = "myst", - subtype: str = "other", -) -> nodes.system_message | None: - """Generate a warning, logging it if necessary. - - If the warning type is listed in the ``suppress_warnings`` configuration, - then ``None`` will be returned and no warning logged. - """ - message = f"{message} [{wtype}.{subtype}]" - kwargs = {"line": line} if line is not None else {} - - if logging.is_suppressed_warning( - wtype, subtype, document.settings.env.app.config.suppress_warnings - ): - return None - - msg_node = document.reporter.warning(message, **kwargs) - if append_to is not None: - append_to.append(msg_node) - - return None - - class SphinxRenderer(DocutilsRenderer): """A markdown-it-py renderer to populate (in-place) a `docutils.document` AST. @@ -61,29 +33,6 @@ class SphinxRenderer(DocutilsRenderer): def doc_env(self) -> BuildEnvironment: return self.document.settings.env - def create_warning( - self, - message: str, - *, - line: int | None = None, - append_to: nodes.Element | None = None, - wtype: str = "myst", - subtype: str = "other", - ) -> nodes.system_message | None: - """Generate a warning, logging it if necessary. - - If the warning type is listed in the ``suppress_warnings`` configuration, - then ``None`` will be returned and no warning logged. - """ - return create_warning( - self.document, - message, - line=line, - append_to=append_to, - wtype=wtype, - subtype=subtype, - ) - def render_internal_link(self, token: SyntaxTreeNode) -> None: """Render link token `[text](link "title")`, where the link has not been identified as an external URL. @@ -171,8 +120,8 @@ def render_heading(self, token: SyntaxTreeNode) -> None: other_doc = self.doc_env.doc2path(domain.labels[doc_slug][0]) self.create_warning( f"duplicate label {doc_slug}, other instance in {other_doc}", + MystWarnings.ANCHOR_DUPE, line=section.line, - subtype="anchor", ) labelid = section["ids"][0] domain.anonlabels[doc_slug] = self.doc_env.docname, labelid diff --git a/myst_parser/parsers/docutils_.py b/myst_parser/parsers/docutils_.py index d0f99bb6..84f4ff92 100644 --- a/myst_parser/parsers/docutils_.py +++ b/myst_parser/parsers/docutils_.py @@ -13,8 +13,9 @@ merge_file_level, read_topmatter, ) -from myst_parser.mdit_to_docutils.base import DocutilsRenderer, create_warning +from myst_parser.mdit_to_docutils.base import DocutilsRenderer from myst_parser.parsers.mdit import create_md_parser +from myst_parser.warnings_ import create_warning def _validate_int( @@ -48,6 +49,10 @@ class Unset: def __repr__(self): return "UNSET" + def __bool__(self): + # this allows to check if the setting is unset/falsy + return False + DOCUTILS_UNSET = Unset() """Sentinel for arguments not set through docutils.conf.""" @@ -218,7 +223,7 @@ def parse(self, inputstring: str, document: nodes.document) -> None: else: if topmatter: warning = lambda wtype, msg: create_warning( # noqa: E731 - document, msg, line=1, append_to=document, subtype=wtype + document, msg, wtype, line=1, append_to=document ) config = merge_file_level(config, topmatter, warning) diff --git a/myst_parser/parsers/sphinx_.py b/myst_parser/parsers/sphinx_.py index fff098f3..94d5aef6 100644 --- a/myst_parser/parsers/sphinx_.py +++ b/myst_parser/parsers/sphinx_.py @@ -12,8 +12,9 @@ merge_file_level, read_topmatter, ) -from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer, create_warning +from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer from myst_parser.parsers.mdit import create_md_parser +from myst_parser.warnings_ import create_warning SPHINX_LOGGER = logging.getLogger(__name__) @@ -60,7 +61,7 @@ def parse(self, inputstring: str, document: nodes.document) -> None: else: if topmatter: warning = lambda wtype, msg: create_warning( # noqa: E731 - document, msg, line=1, append_to=document, subtype=wtype + document, msg, wtype, line=1, append_to=document ) config = merge_file_level(config, topmatter, warning) diff --git a/myst_parser/sphinx_ext/myst_refs.py b/myst_parser/sphinx_ext/myst_refs.py index f70a4de7..948303a3 100644 --- a/myst_parser/sphinx_ext/myst_refs.py +++ b/myst_parser/sphinx_ext/myst_refs.py @@ -17,6 +17,7 @@ from sphinx.util.nodes import clean_astext, make_refnode from myst_parser._compat import findall +from myst_parser.warnings_ import MystWarnings try: from sphinx.errors import NoUri @@ -156,7 +157,7 @@ def resolve_myst_ref( f"Domain '{domain.__module__}::{domain.name}' has not " "implemented a `resolve_any_xref` method [myst.domains]", type="myst", - subtype="domains", + subtype=MystWarnings.LEGACY_DOMAIN.value, once=True, ) for role in domain.roles: @@ -183,7 +184,7 @@ def stringify(name, node): ), location=node, type="myst", - subtype="ref", + subtype=MystWarnings.XREF_AMBIGUOUS.value, ) res_role, newnode = results[0] diff --git a/myst_parser/warnings_.py b/myst_parser/warnings_.py new file mode 100644 index 00000000..4e9bcedd --- /dev/null +++ b/myst_parser/warnings_.py @@ -0,0 +1,101 @@ +"""Central handling of warnings for the myst extension.""" +from __future__ import annotations + +from enum import Enum +from typing import Sequence + +from docutils import nodes + + +class MystWarnings(Enum): + """MyST warning types.""" + + RENDER_METHOD = "render" + """The render method is not implemented.""" + + MD_TOPMATTER = "topmatter" + """Issue reading top-matter.""" + MD_DEF_DUPE = "duplicate_def" + """Duplicate Markdown reference definition.""" + MD_FOOTNOTE_DUPE = "footnote" + """Duplicate Markdown footnote definition.""" + MD_FOOTNOTE_MISSING = "footnote" + """Missing Markdown footnote definition.""" + MD_HEADING_NON_CONSECUTIVE = "header" + """Non-consecutive heading levels.""" + MD_HEADING_NESTED = "nested_header" + """Header found nested in another element.""" + + # cross-reference resolution + XREF_AMBIGUOUS = "xref_ambiguous" + """Multiple targets were found for a cross-reference.""" + LEGACY_DOMAIN = "domains" + """A legacy domain found, which does not support `resolve_any_xref`.""" + + # extensions + ANCHOR_DUPE = "anchor_dupe" + """Duplicate heading anchors generated in same document.""" + STRIKETHROUGH = "strikethrough" + """Strikethrough warning, since only implemented in HTML.""" + HTML_PARSE = "html" + """HTML could not be parsed.""" + INVALID_ATTRIBUTE = "attribute" + """Invalid attribute value.""" + + +def _is_suppressed_warning( + type: str, subtype: str, suppress_warnings: Sequence[str] +) -> bool: + """Check whether the warning is suppressed or not. + + Mirrors: + https://github.com/sphinx-doc/sphinx/blob/47d9035bca9e83d6db30a0726a02dc9265bd66b1/sphinx/util/logging.py + """ + if type is None: + return False + + subtarget: str | None + + for warning_type in suppress_warnings: + if "." in warning_type: + target, subtarget = warning_type.split(".", 1) + else: + target, subtarget = warning_type, None + + if target == type and subtarget in (None, subtype, "*"): + return True + + return False + + +def create_warning( + document: nodes.document, + message: str, + subtype: MystWarnings, + *, + line: int | None = None, + append_to: nodes.Element | None = None, +) -> nodes.system_message | None: + """Generate a warning, logging if it is necessary. + + If the warning type is listed in the ``suppress_warnings`` configuration, + then ``None`` will be returned and no warning logged. + """ + wtype = "myst" + # figure out whether to suppress the warning, if sphinx is available, + # it will have been set up by the Sphinx environment, + # otherwise we will use the configuration set by docutils + suppress_warnings: Sequence[str] = [] + try: + suppress_warnings = document.settings.env.app.config.suppress_warnings + except AttributeError: + suppress_warnings = document.settings.myst_suppress_warnings or [] + if _is_suppressed_warning(wtype, subtype.value, suppress_warnings): + return None + + kwargs = {"line": line} if line is not None else {} + message = f"{message} [{wtype}.{subtype.value}]" + msg_node = document.reporter.warning(message, **kwargs) + if append_to is not None: + append_to.append(msg_node) + return msg_node diff --git a/tests/test_renderers/fixtures/myst-config.txt b/tests/test_renderers/fixtures/myst-config.txt index 668895a2..48fd5e52 100644 --- a/tests/test_renderers/fixtures/myst-config.txt +++ b/tests/test_renderers/fixtures/myst-config.txt @@ -1,3 +1,15 @@ +[suppress-warnings] --myst-suppress-warnings="myst.header" +. +# A +### B +. +<document ids="a" names="a" source="<string>" title="A"> + <title> + A + <subtitle ids="b" names="b"> + B +. + [title-to-header] --myst-title-to-header="yes" . --- @@ -161,16 +173,16 @@ www.commonmark.org/he<lp <paragraph> <system_message level="2" line="1" source="<string>" type="WARNING"> <paragraph> - Invalid width value for image: '1x' [myst.image] + Invalid width value for image: '1x' [myst.attribute] <system_message level="2" line="1" source="<string>" type="WARNING"> <paragraph> - Invalid height value for image: '2x' [myst.image] + Invalid height value for image: '2x' [myst.attribute] <system_message level="2" line="1" source="<string>" type="WARNING"> <paragraph> - Invalid align value for image: 'other' [myst.image] + Invalid align value for image: 'other' [myst.attribute] <image alt="a" uri="b"> -<string>:1: (WARNING/2) Invalid width value for image: '1x' [myst.image] -<string>:1: (WARNING/2) Invalid height value for image: '2x' [myst.image] -<string>:1: (WARNING/2) Invalid align value for image: 'other' [myst.image] +<string>:1: (WARNING/2) Invalid width value for image: '1x' [myst.attribute] +<string>:1: (WARNING/2) Invalid height value for image: '2x' [myst.attribute] +<string>:1: (WARNING/2) Invalid align value for image: 'other' [myst.attribute] . diff --git a/tests/test_renderers/fixtures/reporter_warnings.md b/tests/test_renderers/fixtures/reporter_warnings.md index e9998b90..59a89233 100644 --- a/tests/test_renderers/fixtures/reporter_warnings.md +++ b/tests/test_renderers/fixtures/reporter_warnings.md @@ -3,7 +3,7 @@ Duplicate Reference definitions: [a]: b [a]: c . -<string>:2: (WARNING/2) Duplicate reference definition: A [myst.ref] +<string>:2: (WARNING/2) Duplicate reference definition: A [myst.duplicate_def] . Missing Reference: From bf56662a0109ec25384158e59ea44ad00bac40e8 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Thu, 5 Jan 2023 01:12:40 +0100 Subject: [PATCH 05/20] =?UTF-8?q?=E2=9C=A8=20NEW:=20Add=20`attrs=5Finline`?= =?UTF-8?q?=20extension=20(#654)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By adding `"attrs_inline"` to `myst_enable_extensions` (in the sphinx `conf.py`), you can enable parsing of inline attributes after certain inline syntaxes. This is adapted from [djot inline attributes](https://htmlpreview.github.io/?https://github.com/jgm/djot/blob/master/doc/syntax.html#inline-attributes), and also related to [pandoc bracketed spans](https://pandoc.org/MANUAL.html#extension-bracketed_spans). This extension replaces `"attrs_image"`, which is still present, but deprecated. --- docs/conf.py | 2 +- docs/syntax/optional.md | 128 +++++++++++------ myst_parser/config/main.py | 1 + myst_parser/mdit_to_docutils/base.py | 132 +++++++++++------- myst_parser/mdit_to_docutils/sphinx_.py | 4 +- myst_parser/parsers/mdit.py | 10 +- myst_parser/sphinx_ext/main.py | 10 ++ myst_parser/warnings_.py | 3 + pyproject.toml | 5 +- tests/test_renderers/fixtures/myst-config.txt | 77 ++++++++-- 10 files changed, 256 insertions(+), 116 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7db210c5..e39c8074 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -90,7 +90,7 @@ "strikethrough", "substitution", "tasklist", - "attrs_image", + "attrs_inline", ] myst_number_code_blocks = ["typescript"] myst_heading_anchors = 2 diff --git a/docs/syntax/optional.md b/docs/syntax/optional.md index b11ab08b..c497d929 100644 --- a/docs/syntax/optional.md +++ b/docs/syntax/optional.md @@ -28,6 +28,7 @@ To enable all the syntaxes explained below: ```python myst_enable_extensions = [ "amsmath", + "attrs_inline", "colon_fence", "deflist", "dollarmath", @@ -43,7 +44,7 @@ myst_enable_extensions = [ ] ``` -:::{important} +:::{versionchanged} 0.13.0 `myst_enable_extensions` replaces previous configuration options: `admonition_enable`, `figure_enable`, `dmath_enable`, `amsmath_enable`, `deflist_enable`, `html_img_enable` ::: @@ -101,7 +102,7 @@ Math is parsed by adding to the `myst_enable_extensions` list option, in the sph These options enable their respective Markdown parser plugins, as detailed in the [markdown-it plugin guide](markdown_it:md/plugins). -:::{important} +:::{versionchanged} 0.13.0 `myst_dmath_enable=True` and `myst_amsmath_enable=True` are deprecated, and replaced by `myst_enable_extensions = ["dollarmath", "amsmath"]` ::: @@ -484,7 +485,7 @@ This text is **standard** _Markdown_ ## Admonition directives -:::{important} +:::{versionchanged} 0.13.0 `myst_admonition_enable` is deprecated and replaced by `myst_enable_extensions = ["colon_fence"]` (see above). Also, classes should now be set with the `:class: myclass` option. @@ -727,9 +728,90 @@ Send a message to a recipient Currently `sphinx.ext.autodoc` does not support MyST, see [](howto/autodoc). ::: +(syntax/attributes)= +## Inline attributes + +By adding `"attrs_inline"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), +you can enable parsing of inline attributes after certain inline syntaxes. +This is adapted from [djot inline attributes](https://htmlpreview.github.io/?https://github.com/jgm/djot/blob/master/doc/syntax.html#inline-attributes), +and also related to [pandoc bracketed spans](https://pandoc.org/MANUAL.html#extension-bracketed_spans). + +:::{important} +This feature is in *beta*, and may change in future versions. +It replace the previous `attrs_image` extension, which is now deprecated. +::: + +Attributes are specified in curly braces after the inline syntax. +Inside the curly braces, the following syntax is recognised: + +- `.foo` specifies `foo` as a class. + Multiple classes may be given in this way; they will be combined. +- `#foo` specifies `foo` as an identifier. + An element may have only one identifier; + if multiple identifiers are given, the last one is used. +- `key="value"` or `key=value` specifies a key-value attribute. + Quotes are not needed when the value consists entirely of + ASCII alphanumeric characters or `_` or `:` or `-`. + Backslash escapes may be used inside quoted values. + **Note** only certain keys are supported, see below. +- `%` begins a comment, which ends with the next `%` or the end of the attribute (`}`). + +For example, the following Markdown: + +```md + +- [A span of text with attributes]{#spanid .bg-warning}, + {ref}`a reference to the span <spanid>` + +- `A literal with attributes`{#literalid .bg-warning}, + {ref}`a reference to the literal <literalid> + +- An autolink with attributes: <https://example.com>{.bg-warning} + +- [A link with attributes](syntax/attributes){#linkid .bg-warning}, + {ref}`a reference to the link <linkid>` + +- ![An image with attribute](img/fun-fish.png){#imgid .bg-warning w=100px align=center} + {ref}`a reference to the image <imgid>` + +``` + +will be parsed as: + +- [A span of text with attributes]{#spanid .bg-warning}, + {ref}`a reference to the span <spanid>` + +- `A literal with attributes`{#literalid .bg-warning}, + {ref}`a reference to the literal <literalid>` + +- An autolink with attributes: <https://example.com>{.bg-warning} + +- [A link with attributes](syntax/attributes){#linkid .bg-warning}, + {ref}`a reference to the link <linkid>` + +- ![An image with attribute](img/fun-fish.png){#imgid .bg-warning w="100px" align=center} + {ref}`a reference to the image <imgid>` + +### key-value attributes + +`id` and `class` are supported for all inline syntaxes, +but only certain key-value attributes are supported for each syntax. + +For **literals**, the following attributes are supported: + +- `language`/`lexer`/`l` defines the syntax lexer, + e.g. `` `a = "b"`{l=python} `` is displayed as `a = "b"`{l=python}. + Note, this is only supported in `sphinx >= 5`. + +For **images**, the following attributes are supported (equivalent to the `image` directive): + +- `width`/`w` defines the width of the image (in `%`, `px`, `em`, `cm`, etc) +- `height`/`h` defines the height of the image (in `px`, `em`, `cm`, etc) +- `align`/`a` defines the scale of the image (`left`, `center`, or `right`) + (syntax/images)= -## Images +## HTML Images MyST provides a few different syntaxes for including images in your documentation, as explained below. @@ -786,42 +868,6 @@ HTML image can also be used inline! I'm an inline image: <img src="img/fun-fish.png" height="20px"> -### Inline attributes - -:::{warning} -This extension is currently experimental, and may change in future versions. -::: - -By adding `"attrs_image"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), -you can enable parsing of inline attributes for images. - -For example, the following Markdown: - -```md -![image attrs](img/fun-fish.png){#imgattr .bg-primary width="100px" align=center} - -{ref}`a reference to the image <imgattr>` -``` - -will be parsed as: - -![image attrs](img/fun-fish.png){#imgattr .bg-primary width="100px" align=center} - -{ref}`a reference to the image <imgattr>` - -Inside the curly braces, the following syntax is possible: - -- `.foo` specifies `foo` as a class. - Multiple classes may be given in this way; they will be combined. -- `#foo` specifies `foo` as an identifier. - An element may have only one identifier; - if multiple identifiers are given, the last one is used. -- `key="value"` or `key=value` specifies a key-value attribute. - Quotes are not needed when the value consists entirely of - ASCII alphanumeric characters or `_` or `:` or `-`. - Backslash escapes may be used inside quoted values. -- `%` begins a comment, which ends with the next `%` or the end of the attribute (`}`). - (syntax/figures)= ## Markdown Figures @@ -830,7 +876,7 @@ By adding `"colon_fence"` to `myst_enable_extensions` (in the sphinx `conf.py` [ we can combine the above two extended syntaxes, to create a fully Markdown compliant version of the `figure` directive named `figure-md`. -:::{important} +:::{versionchanged} 0.13.0 `myst_figure_enable` with the `figure` directive is deprecated and replaced by `myst_enable_extensions = ["colon_fence"]` and `figure-md`. ::: diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py index 118c3d08..b32714a5 100644 --- a/myst_parser/config/main.py +++ b/myst_parser/config/main.py @@ -33,6 +33,7 @@ def check_extensions(_, __, value): [ "amsmath", "attrs_image", + "attrs_inline", "colon_fence", "deflist", "dollarmath", diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index f21e6d11..6b198ea9 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -9,7 +9,15 @@ from contextlib import contextmanager from datetime import date, datetime from types import ModuleType -from typing import TYPE_CHECKING, Any, Iterator, MutableMapping, Sequence, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Iterator, + MutableMapping, + Sequence, + cast, +) from urllib.parse import urlparse import jinja2 @@ -362,6 +370,44 @@ def add_line_and_source_path_r( for child in findall(node)(): self.add_line_and_source_path(child, token) + def copy_attributes( + self, + token: SyntaxTreeNode, + node: nodes.Element, + keys: Sequence[str] = ("class",), + *, + converters: dict[str, Callable[[str], Any]] | None = None, + aliases: dict[str, str] | None = None, + ) -> None: + """Copy attributes on the token to the docutils node.""" + if converters is None: + converters = {} + if aliases is None: + aliases = {} + for key, value in token.attrs.items(): + key = aliases.get(key, key) + if key not in keys: + continue + if key == "class": + node["classes"].extend(str(value).split()) + elif key == "id": + name = nodes.fully_normalize_name(str(value)) + node["names"].append(name) + self.document.note_explicit_target(node, node) + else: + if key in converters: + try: + value = converters[key](str(value)) + except ValueError: + self.create_warning( + f"Invalid {key!r} attribute value: {token.attrs[key]!r}", + MystWarnings.INVALID_ATTRIBUTE, + line=token_line(token, default=0), + append_to=node, + ) + continue + node[key] = value + def update_section_level_state(self, section: nodes.section, level: int) -> None: """Update the section level state, with the new current section and level.""" # find the closest parent section @@ -490,6 +536,14 @@ def render_hr(self, token: SyntaxTreeNode) -> None: def render_code_inline(self, token: SyntaxTreeNode) -> None: node = nodes.literal(token.content, token.content) self.add_line_and_source_path(node, token) + self.copy_attributes( + token, + node, + ("class", "id", "language"), + aliases={"lexer": "language", "l": "language"}, + ) + if "language" in node and "code" not in node["classes"]: + node["classes"].append("code") self.current_node.append(node) def create_highlighted_code_block( @@ -697,10 +751,8 @@ def render_external_url(self, token: SyntaxTreeNode) -> None: """ ref_node = nodes.reference() self.add_line_and_source_path(ref_node, token) + self.copy_attributes(token, ref_node, ("class", "id", "title")) ref_node["refuri"] = cast(str, token.attrGet("href") or "") - title = token.attrGet("title") - if title: - ref_node["title"] = title with self.current_node_context(ref_node, append=True): self.render_children(token) @@ -717,17 +769,16 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: """ ref_node = nodes.reference() self.add_line_and_source_path(ref_node, token) + self.copy_attributes(token, ref_node, ("class", "id", "title")) ref_node["refname"] = cast(str, token.attrGet("href") or "") self.document.note_refname(ref_node) - title = token.attrGet("title") - if title: - ref_node["title"] = title with self.current_node_context(ref_node, append=True): self.render_children(token) def render_autolink(self, token: SyntaxTreeNode) -> None: refuri = escapeHtml(token.attrGet("href") or "") # type: ignore[arg-type] ref_node = nodes.reference() + self.copy_attributes(token, ref_node, ("class", "id")) ref_node["refuri"] = refuri self.add_line_and_source_path(ref_node, token) with self.current_node_context(ref_node, append=True): @@ -760,58 +811,31 @@ def render_image(self, token: SyntaxTreeNode) -> None: img_node["uri"] = destination img_node["alt"] = self.renderInlineAsText(token.children or []) - title = token.attrGet("title") - if title: - img_node["title"] = token.attrGet("title") - - # apply other attributes that can be set on the image - if "class" in token.attrs: - img_node["classes"].extend(str(token.attrs["class"]).split()) - if "width" in token.attrs: - try: - width = directives.length_or_percentage_or_unitless( - str(token.attrs["width"]) - ) - except ValueError: - self.create_warning( - f"Invalid width value for image: {token.attrs['width']!r}", - MystWarnings.INVALID_ATTRIBUTE, - line=token_line(token, default=0), - append_to=self.current_node, - ) - else: - img_node["width"] = width - if "height" in token.attrs: - try: - height = directives.length_or_unitless(str(token.attrs["height"])) - except ValueError: - self.create_warning( - f"Invalid height value for image: {token.attrs['height']!r}", - MystWarnings.INVALID_ATTRIBUTE, - line=token_line(token, default=0), - append_to=self.current_node, - ) - else: - img_node["height"] = height - if "align" in token.attrs: - if token.attrs["align"] not in ("left", "center", "right"): - self.create_warning( - f"Invalid align value for image: {token.attrs['align']!r}", - MystWarnings.INVALID_ATTRIBUTE, - line=token_line(token, default=0), - append_to=self.current_node, - ) - else: - img_node["align"] = token.attrs["align"] - if "id" in token.attrs: - name = nodes.fully_normalize_name(str(token.attrs["id"])) - img_node["names"].append(name) - self.document.note_explicit_target(img_node, img_node) + + self.copy_attributes( + token, + img_node, + ("class", "id", "title", "width", "height", "align"), + converters={ + "width": directives.length_or_percentage_or_unitless, + "height": directives.length_or_unitless, + "align": lambda x: directives.choice(x, ("left", "center", "right")), + }, + aliases={"w": "width", "h": "height", "a": "align"}, + ) self.current_node.append(img_node) # ### render methods for plugin tokens + def render_span(self, token: SyntaxTreeNode) -> None: + """Render an inline span token.""" + node = nodes.inline() + self.add_line_and_source_path(node, token) + self.copy_attributes(token, node, ("class", "id")) + with self.current_node_context(node, append=True): + self.render_children(token) + def render_front_matter(self, token: SyntaxTreeNode) -> None: """Pass document front matter data.""" position = token_line(token, default=0) diff --git a/myst_parser/mdit_to_docutils/sphinx_.py b/myst_parser/mdit_to_docutils/sphinx_.py index b6c55726..b6607f30 100644 --- a/myst_parser/mdit_to_docutils/sphinx_.py +++ b/myst_parser/mdit_to_docutils/sphinx_.py @@ -84,9 +84,7 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: text = "" self.add_line_and_source_path(wrap_node, token) - title = token.attrGet("title") - if title: - wrap_node["title"] = title + self.copy_attributes(token, wrap_node, ("class", "id", "title")) self.current_node.append(wrap_node) inner_node = nodes.inline("", text, classes=classes) diff --git a/myst_parser/parsers/mdit.py b/myst_parser/parsers/mdit.py index 84764957..dbb533d4 100644 --- a/myst_parser/parsers/mdit.py +++ b/myst_parser/parsers/mdit.py @@ -101,7 +101,15 @@ def create_md_parser( md.use(tasklists_plugin) if "substitution" in config.enable_extensions: md.use(substitution_plugin, *config.sub_delimiters) - if "attrs_image" in config.enable_extensions: + if "attrs_inline" in config.enable_extensions: + md.use( + attrs_plugin, + after=("image", "code_inline", "link_close", "span_close"), + spans=True, + span_after="footnote_ref", + ) + elif "attrs_image" in config.enable_extensions: + # TODO deprecate md.use(attrs_plugin, after=("image",)) if config.heading_anchors is not None: md.use( diff --git a/myst_parser/sphinx_ext/main.py b/myst_parser/sphinx_ext/main.py index f5aeffc1..c7e2a672 100644 --- a/myst_parser/sphinx_ext/main.py +++ b/myst_parser/sphinx_ext/main.py @@ -3,6 +3,8 @@ from sphinx.application import Sphinx +from myst_parser.warnings_ import MystWarnings + def setup_sphinx(app: Sphinx, load_parser=False): """Initialize all settings and transforms in Sphinx.""" @@ -58,3 +60,11 @@ def create_myst_config(app): except (TypeError, ValueError) as error: logger.error("myst configuration invalid: %s", error.args[0]) app.env.myst_config = MdParserConfig() + + if "attrs_image" in app.env.myst_config.enable_extensions: + logger.warning( + "The `attrs_image` extension is deprecated, " + "please use `attrs_inline` instead.", + type="myst", + subtype=MystWarnings.DEPRECATED.value, + ) diff --git a/myst_parser/warnings_.py b/myst_parser/warnings_.py index 4e9bcedd..78b94914 100644 --- a/myst_parser/warnings_.py +++ b/myst_parser/warnings_.py @@ -10,6 +10,9 @@ class MystWarnings(Enum): """MyST warning types.""" + DEPRECATED = "deprecated" + """Deprecated usage.""" + RENDER_METHOD = "render" """The render method is not implemented.""" diff --git a/pyproject.toml b/pyproject.toml index 82cfb5e9..1e71cf66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "docutils>=0.15,<0.20", "jinja2", # required for substitutions, but let sphinx choose version "markdown-it-py>=1.0.0,<3.0.0", - "mdit-py-plugins~=0.3.1", + "mdit-py-plugins~=0.3.3", "pyyaml", "sphinx>=4,<6", 'typing-extensions; python_version < "3.8"', @@ -54,7 +54,8 @@ linkify = ["linkify-it-py~=1.0"] # Note: This is only required for internal use rtd = [ "ipython", - "sphinx-book-theme", + # currently required to get sphinx v5 + "sphinx-book-theme @ git+https://github.com/executablebooks/sphinx-book-theme.git@8da268fce3159755041e8db93e132221a0b0def5#egg=sphinx-book-theme", "sphinx-design", "sphinxext-rediraffe~=0.2.7", "sphinxcontrib.mermaid~=0.7.1", diff --git a/tests/test_renderers/fixtures/myst-config.txt b/tests/test_renderers/fixtures/myst-config.txt index 48fd5e52..8a71971f 100644 --- a/tests/test_renderers/fixtures/myst-config.txt +++ b/tests/test_renderers/fixtures/myst-config.txt @@ -156,7 +156,56 @@ www.commonmark.org/he<lp <lp . -[attrs_image] --myst-enable-extensions=attrs_image +[attrs_inline_span] --myst-enable-extensions=attrs_inline +. +[content]{#id .a .b} +. +<document source="<string>"> + <paragraph> + <inline classes="a b" ids="id" names="id"> + content +. + +[attrs_inline_code] --myst-enable-extensions=attrs_inline +. +`content`{#id .a .b language=python} +. +<document source="<string>"> + <paragraph> + <literal classes="a b code" ids="id" language="python" names="id"> + content +. + +[attrs_inline_links] --myst-enable-extensions=attrs_inline +. +<https://example.com>{.a .b} + +(other)= +[text1](https://example.com){#id1 .a .b} + +[text2](other){#id2 .c .d} + +[ref]{#id3 .e .f} + +[ref]: https://example.com +. +<document source="<string>"> + <paragraph> + <reference classes="a b" refuri="https://example.com"> + https://example.com + <target refid="other"> + <paragraph ids="other" names="other"> + <reference classes="a b" ids="id1" names="id1" refuri="https://example.com"> + text1 + <paragraph> + <reference classes="c d" ids="id2" names="id2" refid="other"> + text2 + <paragraph> + <reference classes="e f" ids="id3" names="id3" refuri="https://example.com"> + ref +. + +[attrs_inline_image] --myst-enable-extensions=attrs_inline . ![a](b){#id .a width="100%" align=center height=20px}{.b} . @@ -165,24 +214,24 @@ www.commonmark.org/he<lp <image align="center" alt="a" classes="a b" height="20px" ids="id" names="id" uri="b" width="100%"> . -[attrs_image_warnings] --myst-enable-extensions=attrs_image +[attrs_inline_image_warnings] --myst-enable-extensions=attrs_inline . ![a](b){width=1x height=2x align=other } . <document source="<string>"> <paragraph> - <system_message level="2" line="1" source="<string>" type="WARNING"> - <paragraph> - Invalid width value for image: '1x' [myst.attribute] - <system_message level="2" line="1" source="<string>" type="WARNING"> - <paragraph> - Invalid height value for image: '2x' [myst.attribute] - <system_message level="2" line="1" source="<string>" type="WARNING"> - <paragraph> - Invalid align value for image: 'other' [myst.attribute] <image alt="a" uri="b"> + <system_message level="2" line="1" source="<string>" type="WARNING"> + <paragraph> + Invalid 'width' attribute value: '1x' [myst.attribute] + <system_message level="2" line="1" source="<string>" type="WARNING"> + <paragraph> + Invalid 'height' attribute value: '2x' [myst.attribute] + <system_message level="2" line="1" source="<string>" type="WARNING"> + <paragraph> + Invalid 'align' attribute value: 'other' [myst.attribute] -<string>:1: (WARNING/2) Invalid width value for image: '1x' [myst.attribute] -<string>:1: (WARNING/2) Invalid height value for image: '2x' [myst.attribute] -<string>:1: (WARNING/2) Invalid align value for image: 'other' [myst.attribute] +<string>:1: (WARNING/2) Invalid 'width' attribute value: '1x' [myst.attribute] +<string>:1: (WARNING/2) Invalid 'height' attribute value: '2x' [myst.attribute] +<string>:1: (WARNING/2) Invalid 'align' attribute value: 'other' [myst.attribute] . From 2a206a8777a004c9be3c6138ed8c7505771c8567 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Thu, 5 Jan 2023 02:01:20 +0100 Subject: [PATCH 06/20] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Drop=20sphinx=204=20?= =?UTF-8?q?support,=20add=20sphinx=206=20(#664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tests.yml | 14 +- pyproject.toml | 2 +- tests/test_sphinx/test_sphinx_builds.py | 9 +- ...est_basic.sphinx5.html => test_basic.html} | 0 .../test_basic.sphinx4.html | 252 ------------------ ...nx5.html => test_fieldlist_extension.html} | 0 .../test_fieldlist_extension.sphinx4.html | 131 --------- ...notes.sphinx5.html => test_footnotes.html} | 0 .../test_footnotes.sphinx4.html | 147 ---------- ...ml.sphinx5.html => test_gettext_html.html} | 0 .../test_gettext_html.sphinx4.html | 162 ----------- tox.ini | 10 +- 12 files changed, 15 insertions(+), 712 deletions(-) rename tests/test_sphinx/test_sphinx_builds/{test_basic.sphinx5.html => test_basic.html} (100%) delete mode 100644 tests/test_sphinx/test_sphinx_builds/test_basic.sphinx4.html rename tests/test_sphinx/test_sphinx_builds/{test_fieldlist_extension.sphinx5.html => test_fieldlist_extension.html} (100%) delete mode 100644 tests/test_sphinx/test_sphinx_builds/test_fieldlist_extension.sphinx4.html rename tests/test_sphinx/test_sphinx_builds/{test_footnotes.sphinx5.html => test_footnotes.html} (100%) delete mode 100644 tests/test_sphinx/test_sphinx_builds/test_footnotes.sphinx4.html rename tests/test_sphinx/test_sphinx_builds/{test_gettext_html.sphinx5.html => test_gettext_html.html} (100%) delete mode 100644 tests/test_sphinx/test_sphinx_builds/test_gettext_html.sphinx4.html diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 208652db..6ca1ae05 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,16 +25,16 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - sphinx: [">=5,<6"] + python-version: ["3.8", "3.9", "3.10", "3.11"] + sphinx: [">=6,<7"] os: [ubuntu-latest] include: - os: ubuntu-latest python-version: "3.8" - sphinx: ">=4,<5" + sphinx: ">=5,<6" - os: windows-latest python-version: "3.8" - sphinx: ">=4,<5" + sphinx: ">=5,<6" runs-on: ${{ matrix.os }} @@ -47,8 +47,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e .[linkify,testing] - pip install --upgrade-strategy "only-if-needed" "sphinx${{ matrix.sphinx }}" + pip install -e ".[linkify,testing]" "sphinx${{ matrix.sphinx }}" - name: Run pytest run: | pytest --cov=myst_parser --cov-report=xml --cov-report=term-missing @@ -87,8 +86,7 @@ jobs: run: python .github/workflows/docutils_setup.py pyproject.toml README.md - name: Install dependencies run: | - pip install . - pip install pytest~=6.2 pytest-param-files~=0.3.3 pygments docutils==${{ matrix.docutils-version }} + pip install . pytest~=6.2 pytest-param-files~=0.3.3 pygments docutils==${{ matrix.docutils-version }} - name: ensure sphinx is not installed run: | python -c "\ diff --git a/pyproject.toml b/pyproject.toml index 1e71cf66..b3b3a90e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "markdown-it-py>=1.0.0,<3.0.0", "mdit-py-plugins~=0.3.3", "pyyaml", - "sphinx>=4,<6", + "sphinx>=5,<7", 'typing-extensions; python_version < "3.8"', ] diff --git a/tests/test_sphinx/test_sphinx_builds.py b/tests/test_sphinx/test_sphinx_builds.py index 0d45a424..8ede05d7 100644 --- a/tests/test_sphinx/test_sphinx_builds.py +++ b/tests/test_sphinx/test_sphinx_builds.py @@ -11,7 +11,6 @@ import re import pytest -import sphinx from docutils import VersionInfo, __version_info__ SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "sourcedirs")) @@ -54,7 +53,7 @@ def test_basic( app, filename="content.html", regress_html=True, - regress_ext=f".sphinx{sphinx.version_info[0]}.html", + regress_ext=".html", ) assert app.env.metadata["content"] == { @@ -333,7 +332,7 @@ def test_footnotes( app, filename="footnote_md.html", regress_html=True, - regress_ext=f".sphinx{sphinx.version_info[0]}.html", + regress_ext=".html", ) @@ -454,7 +453,7 @@ def test_gettext_html( app, filename="index.html", regress_html=True, - regress_ext=f".sphinx{sphinx.version_info[0]}.html", + regress_ext=".html", ) @@ -555,7 +554,7 @@ def test_fieldlist_extension( app, filename="index.html", regress_html=True, - regress_ext=f".sphinx{sphinx.version_info[0]}.html", + regress_ext=".html", ) diff --git a/tests/test_sphinx/test_sphinx_builds/test_basic.sphinx5.html b/tests/test_sphinx/test_sphinx_builds/test_basic.html similarity index 100% rename from tests/test_sphinx/test_sphinx_builds/test_basic.sphinx5.html rename to tests/test_sphinx/test_sphinx_builds/test_basic.html diff --git a/tests/test_sphinx/test_sphinx_builds/test_basic.sphinx4.html b/tests/test_sphinx/test_sphinx_builds/test_basic.sphinx4.html deleted file mode 100644 index 743f7a19..00000000 --- a/tests/test_sphinx/test_sphinx_builds/test_basic.sphinx4.html +++ /dev/null @@ -1,252 +0,0 @@ -<div class="documentwrapper"> - <div class="bodywrapper"> - <div class="body" role="main"> - <div class="dedication topic"> - <p class="topic-title"> - Dedication - </p> - <p> - To my - <em> - homies - </em> - </p> - </div> - <div class="abstract topic"> - <p class="topic-title"> - Abstract - </p> - <p> - Something something - <strong> - dark - </strong> - side - </p> - </div> - <section class="tex2jax_ignore mathjax_ignore" id="header"> - <span id="target"> - </span> - <h1> - Header - <a class="headerlink" href="#header" title="Permalink to this headline"> - ¶ - </a> - </h1> - <div class="admonition note"> - <p class="admonition-title"> - Note - </p> - <p> - abcd - <em> - abc - </em> - <a class="reference external" href="https://www.google.com"> - google - </a> - </p> - <div class="admonition warning"> - <p class="admonition-title"> - Warning - </p> - <p> - xyz - </p> - </div> - </div> - <div class="admonition-title-with-link-target2 admonition"> - <p class="admonition-title"> - Title with - <a class="reference internal" href="#target2"> - <span class="std std-ref"> - link - </span> - </a> - </p> - <p> - Content - </p> - </div> - <figure class="align-default" id="id1"> - <span id="target2"> - </span> - <a class="reference external image-reference" href="https://www.google.com"> - <img alt="_images/example.jpg" src="_images/example.jpg" style="height: 40px;"/> - </a> - <figcaption> - <p> - <span class="caption-text"> - Caption - </span> - <a class="headerlink" href="#id1" title="Permalink to this image"> - ¶ - </a> - </p> - </figcaption> - </figure> - <p> - <img alt="alternative text" src="_images/example.jpg"/> - </p> - <p> - <a class="reference external" href="https://www.google.com"> - https://www.google.com - </a> - </p> - <p> - <strong> - <code class="code docutils literal notranslate"> - <span class="pre"> - a=1{`} - </span> - </code> - </strong> - </p> - <p> - <span class="math notranslate nohighlight"> - \(sdfds\) - </span> - </p> - <p> - <strong> - <span class="math notranslate nohighlight"> - \(a=1\) - </span> - </strong> - </p> - <div class="math notranslate nohighlight"> - \[b=2\] - </div> - <div class="math notranslate nohighlight" id="equation-eq-label"> - <span class="eqno"> - (1) - <a class="headerlink" href="#equation-eq-label" title="Permalink to this equation"> - ¶ - </a> - </span> - \[c=2\] - </div> - <p> - <a class="reference internal" href="#equation-eq-label"> - (1) - </a> - </p> - <p> - <code class="docutils literal notranslate"> - <span class="pre"> - a=1{`} - </span> - </code> - </p> - <table class="colwidths-auto docutils align-default"> - <thead> - <tr class="row-odd"> - <th class="head"> - <p> - a - </p> - </th> - <th class="text-right head"> - <p> - b - </p> - </th> - </tr> - </thead> - <tbody> - <tr class="row-even"> - <td> - <p> - <em> - a - </em> - </p> - </td> - <td class="text-right"> - <p> - 2 - </p> - </td> - </tr> - <tr class="row-odd"> - <td> - <p> - <a class="reference external" href="https://google.com"> - link-a - </a> - </p> - </td> - <td class="text-right"> - <p> - <a class="reference external" href="https://python.org"> - link-b - </a> - </p> - </td> - </tr> - </tbody> - </table> - <p> - this -is -a -paragraph - </p> - <p> - this is a second paragraph - </p> - <ul class="simple"> - <li> - <p> - a list - </p> - <ul> - <li> - <p> - a sub list - </p> - </li> - </ul> - </li> - </ul> - <ul class="simple"> - <li> - <p> - new list? - </p> - </li> - </ul> - <p> - <a class="reference internal" href="#target"> - <span class="std std-ref"> - Header - </span> - </a> - <a class="reference internal" href="#target2"> - <span class="std std-ref"> - Caption - </span> - </a> - </p> - <p> - <a class="reference external" href="https://www.google.com"> - name - </a> - </p> - <div class="highlight-default notranslate"> - <div class="highlight"> - <pre><span></span><span class="k">def</span> <span class="nf">func</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span> - <span class="nb">print</span><span class="p">(</span><span class="n">a</span><span class="p">)</span> -</pre> - </div> - </div> - <p> - Special substitution references: - </p> - <p> - 57 words | 0 min read - </p> - </section> - </div> - </div> -</div> diff --git a/tests/test_sphinx/test_sphinx_builds/test_fieldlist_extension.sphinx5.html b/tests/test_sphinx/test_sphinx_builds/test_fieldlist_extension.html similarity index 100% rename from tests/test_sphinx/test_sphinx_builds/test_fieldlist_extension.sphinx5.html rename to tests/test_sphinx/test_sphinx_builds/test_fieldlist_extension.html diff --git a/tests/test_sphinx/test_sphinx_builds/test_fieldlist_extension.sphinx4.html b/tests/test_sphinx/test_sphinx_builds/test_fieldlist_extension.sphinx4.html deleted file mode 100644 index bef86e92..00000000 --- a/tests/test_sphinx/test_sphinx_builds/test_fieldlist_extension.sphinx4.html +++ /dev/null @@ -1,131 +0,0 @@ -<div class="documentwrapper"> - <div class="bodywrapper"> - <div class="body" role="main"> - <section id="test"> - <h1> - Test - <a class="headerlink" href="#test" title="Permalink to this headline"> - ¶ - </a> - </h1> - <dl class="myst field-list simple"> - <dt class="field-odd"> - field - </dt> - <dd class="field-odd"> - <p> - </p> - </dd> - <dt class="field-even"> - <em> - field - </em> - </dt> - <dd class="field-even"> - <p> - content - </p> - </dd> - </dl> - <dl class="py function"> - <dt class="sig sig-object py" id="send_message"> - <span class="sig-name descname"> - <span class="pre"> - send_message - </span> - </span> - <span class="sig-paren"> - ( - </span> - <em class="sig-param"> - <span class="n"> - <span class="pre"> - sender - </span> - </span> - </em> - , - <em class="sig-param"> - <span class="n"> - <span class="pre"> - priority - </span> - </span> - </em> - <span class="sig-paren"> - ) - </span> - <a class="headerlink" href="#send_message" title="Permalink to this definition"> - ¶ - </a> - </dt> - <dd> - <p> - Send a message to a recipient - </p> - <dl class="myst field-list simple"> - <dt class="field-odd"> - Parameters - </dt> - <dd class="field-odd"> - <ul class="simple"> - <li> - <p> - <strong> - sender - </strong> - ( - <em> - str - </em> - ) – The person sending the message - </p> - </li> - <li> - <p> - <strong> - priority - </strong> - ( - <em> - int - </em> - ) – The priority of the message, can be a number 1-5 - </p> - </li> - </ul> - </dd> - <dt class="field-even"> - Returns - </dt> - <dd class="field-even"> - <p> - the message id - </p> - </dd> - <dt class="field-odd"> - Return type - </dt> - <dd class="field-odd"> - <p> - int - </p> - </dd> - <dt class="field-even"> - Raises - </dt> - <dd class="field-even"> - <p> - <strong> - ValueError - </strong> - – if the message_body exceeds 160 characters - </p> - </dd> - </dl> - </dd> - </dl> - </section> - </div> - </div> -</div> diff --git a/tests/test_sphinx/test_sphinx_builds/test_footnotes.sphinx5.html b/tests/test_sphinx/test_sphinx_builds/test_footnotes.html similarity index 100% rename from tests/test_sphinx/test_sphinx_builds/test_footnotes.sphinx5.html rename to tests/test_sphinx/test_sphinx_builds/test_footnotes.html diff --git a/tests/test_sphinx/test_sphinx_builds/test_footnotes.sphinx4.html b/tests/test_sphinx/test_sphinx_builds/test_footnotes.sphinx4.html deleted file mode 100644 index 70cfb540..00000000 --- a/tests/test_sphinx/test_sphinx_builds/test_footnotes.sphinx4.html +++ /dev/null @@ -1,147 +0,0 @@ -<div class="documentwrapper"> - <div class="bodywrapper"> - <div class="body" role="main"> - <section id="footnotes-with-markdown"> - <h1> - Footnotes with Markdown - <a class="headerlink" href="#footnotes-with-markdown" title="Permalink to this headline"> - ¶ - </a> - </h1> - <p> - <a class="footnote-reference brackets" href="#c" id="id1"> - 1 - </a> - </p> - <div class="admonition note"> - <p class="admonition-title"> - Note - </p> - <p> - <a class="footnote-reference brackets" href="#d" id="id2"> - 2 - </a> - </p> - </div> - <p> - <a class="footnote-reference brackets" href="#a" id="id3"> - 3 - </a> - </p> - <p> - <a class="footnote-reference brackets" href="#b" id="id4"> - 4 - </a> - </p> - <p> - <a class="footnote-reference brackets" href="#id8" id="id5"> - 123 - </a> - <a class="footnote-reference brackets" href="#id8" id="id6"> - 123 - </a> - </p> - <p> - <a class="footnote-reference brackets" href="#e" id="id7"> - 5 - </a> - </p> - <blockquote> - <div> - <ul class="simple"> - <li> - </li> - </ul> - </div> - </blockquote> - <hr class="footnotes docutils"/> - <dl class="footnote brackets"> - <dt class="label" id="c"> - <span class="brackets"> - <a class="fn-backref" href="#id1"> - 1 - </a> - </span> - </dt> - <dd> - <p> - a footnote referenced first - </p> - </dd> - <dt class="label" id="d"> - <span class="brackets"> - <a class="fn-backref" href="#id2"> - 2 - </a> - </span> - </dt> - <dd> - <p> - a footnote referenced in a directive - </p> - </dd> - <dt class="label" id="a"> - <span class="brackets"> - <a class="fn-backref" href="#id3"> - 3 - </a> - </span> - </dt> - <dd> - <p> - some footnote - <em> - text - </em> - </p> - </dd> - <dt class="label" id="b"> - <span class="brackets"> - <a class="fn-backref" href="#id4"> - 4 - </a> - </span> - </dt> - <dd> - <p> - a footnote before its reference - </p> - </dd> - <dt class="label" id="id8"> - <span class="brackets"> - 123 - </span> - <span class="fn-backref"> - ( - <a href="#id5"> - 1 - </a> - , - <a href="#id6"> - 2 - </a> - ) - </span> - </dt> - <dd> - <p> - multiple references footnote - </p> - </dd> - <dt class="label" id="e"> - <span class="brackets"> - <a class="fn-backref" href="#id7"> - 5 - </a> - </span> - </dt> - <dd> - <p> - footnote definition in a block element - </p> - </dd> - </dl> - </section> - </div> - </div> -</div> diff --git a/tests/test_sphinx/test_sphinx_builds/test_gettext_html.sphinx5.html b/tests/test_sphinx/test_sphinx_builds/test_gettext_html.html similarity index 100% rename from tests/test_sphinx/test_sphinx_builds/test_gettext_html.sphinx5.html rename to tests/test_sphinx/test_sphinx_builds/test_gettext_html.html diff --git a/tests/test_sphinx/test_sphinx_builds/test_gettext_html.sphinx4.html b/tests/test_sphinx/test_sphinx_builds/test_gettext_html.sphinx4.html deleted file mode 100644 index 825048a1..00000000 --- a/tests/test_sphinx/test_sphinx_builds/test_gettext_html.sphinx4.html +++ /dev/null @@ -1,162 +0,0 @@ -<div class="documentwrapper"> - <div class="bodywrapper"> - <div class="body" role="main"> - <section id="bold-text-1"> - <h1> - texte 1 en - <strong> - gras - </strong> - <a class="headerlink" href="#bold-text-1" title="Lien permanent vers ce titre"> - ¶ - </a> - </h1> - <p> - texte 2 en - <strong> - gras - </strong> - </p> - <blockquote> - <div> - <p> - texte 3 en - <strong> - gras - </strong> - </p> - </div> - </blockquote> - <div class="admonition note"> - <p class="admonition-title"> - Note - </p> - <p> - texte 4 en - <strong> - gras - </strong> - </p> - </div> - <ul class="simple"> - <li> - <p> - texte 5 en - <strong> - gras - </strong> - </p> - </li> - </ul> - <ol class="arabic simple"> - <li> - <p> - texte 6 en - <strong> - gras - </strong> - </p> - </li> - </ol> - <dl class="simple myst"> - <dt> - texte 7 en - <strong> - gras - </strong> - </dt> - <dd> - <p> - texte 8 en - <strong> - gras - </strong> - </p> - </dd> - </dl> - <table class="colwidths-auto docutils align-default"> - <thead> - <tr class="row-odd"> - <th class="head"> - <p> - texte 9 en - <strong> - gras - </strong> - </p> - </th> - </tr> - </thead> - <tbody> - <tr class="row-even"> - <td> - <p> - texte 10 en - <strong> - gras - </strong> - </p> - </td> - </tr> - </tbody> - </table> - <div markdown="1"> - <p> - texte 11 en - <strong> - gras - </strong> - </p> - <p> - « - <code class="docutils literal notranslate"> - <span class="pre"> - Backtick - </span> - </code> - » supplémentaire - </p> - </div> - <div class="highlight-none notranslate"> - <div class="highlight"> - <pre><span></span>**additional** text 12 -</pre> - </div> - </div> - <div class="highlight-default notranslate"> - <div class="highlight"> - <pre><span></span><span class="o">**</span><span class="n">additional</span><span class="o">**</span> <span class="n">text</span> <span class="mi">13</span> -</pre> - </div> - </div> - <div class="highlight-json notranslate"> - <div class="highlight"> - <pre><span></span><span class="p">{</span> - <span class="nt">"additional"</span><span class="p">:</span> <span class="s2">"text 14"</span> -<span class="p">}</span> -</pre> - </div> - </div> - <h3> - **additional** text 15 - </h3> - <div class="highlight-python notranslate"> - <div class="highlight"> - <pre><span></span><span class="gp">>>> </span><span class="nb">print</span><span class="p">(</span><span class="s1">'doctest block'</span><span class="p">)</span> -<span class="go">doctest block</span> -</pre> - </div> - </div> - <iframe src="http://sphinx-doc.org"> - </iframe> - <p> - <img alt="Poisson amusant 1" src="_images/poisson-amusant.png"/> - </p> - <img alt="Poisson amusant 2" src="_images/fun-fish.png"/> - <figure class="align-default"> - <img alt="Poisson amusant 3" src="_images/fun-fish.png"/> - </figure> - </section> - </div> - </div> -</div> diff --git a/tox.ini b/tox.ini index 4a0110e6..17a3e1c7 100644 --- a/tox.ini +++ b/tox.ini @@ -11,20 +11,18 @@ # then then deleting compiled files has been found to fix it: `find . -name \*.pyc -delete` [tox] -envlist = py37-sphinx5 +envlist = py38-sphinx6 [testenv] usedevelop = true -[testenv:py{37,38,39,310}-sphinx{4,5}] +[testenv:py{37,38,39,310,311}-sphinx{5,6}] deps = - black - flake8 + sphinx5: sphinx>=5,<6 + sphinx6: sphinx>=6,<7 extras = linkify testing -commands_pre = - sphinx4: pip install --quiet --upgrade-strategy "only-if-needed" "sphinx==4.5.0" commands = pytest {posargs} [testenv:docs-{update,clean}] From 23ae1350b0f3796cd51b9f5b129a85ec324352e2 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Thu, 5 Jan 2023 18:54:59 +0100 Subject: [PATCH 07/20] =?UTF-8?q?=F0=9F=A7=AA=20Add=20testing=20of=20docum?= =?UTF-8?q?ent=20builds=20in=20other=20formats=20(#665)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test-formats.yml | 76 +++++++++++++++ docs/conf.py | 142 ++++++++++++++++------------- docs/syntax/optional.md | 18 ++-- tox.ini | 2 +- 4 files changed, 162 insertions(+), 76 deletions(-) create mode 100644 .github/workflows/test-formats.yml diff --git a/.github/workflows/test-formats.yml b/.github/workflows/test-formats.yml new file mode 100644 index 00000000..3bb31c6e --- /dev/null +++ b/.github/workflows/test-formats.yml @@ -0,0 +1,76 @@ +name: build-doc-formats + +on: + push: + branches: [master] + pull_request: + +jobs: + + doc-builds: + + name: Documentation builds + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + format: ["man", "text"] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[linkify,rtd] + - name: Build docs + run: | + sphinx-build -nW --keep-going -b ${{ matrix.format }} docs/ docs/_build/${{ matrix.format }} + + doc-builds-pdf: + + name: Documentation builds + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + format: ["latex"] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[linkify,rtd] + - name: Build docs + run: | + sphinx-build -nW --keep-going -b ${{ matrix.format }} docs/ docs/_build/${{ matrix.format }} + + - name: Make PDF + uses: xu-cheng/latex-action@v2 + with: + working_directory: docs/_build/latex + root_file: "mystparser.tex" + # https://github.com/marketplace/actions/github-action-for-latex#it-fails-due-to-xindy-cannot-be-found + pre_compile: | + ln -sf /opt/texlive/texdir/texmf-dist/scripts/xindy/xindy.pl /opt/texlive/texdir/bin/x86_64-linuxmusl/xindy + ln -sf /opt/texlive/texdir/texmf-dist/scripts/xindy/texindy.pl /opt/texlive/texdir/bin/x86_64-linuxmusl/texindy + wget https://sourceforge.net/projects/xindy/files/xindy-source-components/2.4/xindy-kernel-3.0.tar.gz + tar xf xindy-kernel-3.0.tar.gz + cd xindy-kernel-3.0/src + apk add make + apk add clisp --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community + make + cp -f xindy.mem /opt/texlive/texdir/bin/x86_64-linuxmusl/ + cd ../../ + env: + XINDYOPTS: -L english -C utf8 -M sphinx.xdy diff --git a/docs/conf.py b/docs/conf.py index e39c8074..301fa1f1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,6 +7,7 @@ from datetime import date from sphinx.application import Sphinx +from sphinx.transforms.post_transforms import SphinxPostTransform from myst_parser import __version__ @@ -44,12 +45,66 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +suppress_warnings = ["myst.strikethrough"] -# -- Options for HTML output ------------------------------------------------- +intersphinx_mapping = { + "python": ("https://docs.python.org/3.7", None), + "sphinx": ("https://www.sphinx-doc.org/en/master", None), + "markdown_it": ("https://markdown-it-py.readthedocs.io/en/latest", None), +} + +# -- Autodoc settings --------------------------------------------------- + +autodoc_member_order = "bysource" +nitpicky = True +nitpick_ignore = [ + ("py:class", "docutils.nodes.document"), + ("py:class", "docutils.nodes.docinfo"), + ("py:class", "docutils.nodes.Element"), + ("py:class", "docutils.nodes.Node"), + ("py:class", "docutils.nodes.field_list"), + ("py:class", "docutils.nodes.problematic"), + ("py:class", "docutils.nodes.pending"), + ("py:class", "docutils.nodes.system_message"), + ("py:class", "docutils.statemachine.StringList"), + ("py:class", "docutils.parsers.rst.directives.misc.Include"), + ("py:class", "docutils.parsers.rst.Parser"), + ("py:class", "docutils.utils.Reporter"), + ("py:class", "nodes.Element"), + ("py:class", "nodes.Node"), + ("py:class", "nodes.system_message"), + ("py:class", "Directive"), + ("py:class", "Include"), + ("py:class", "StringList"), + ("py:class", "DocutilsRenderer"), + ("py:class", "MockStateMachine"), +] + +# -- MyST settings --------------------------------------------------- + +myst_enable_extensions = [ + "dollarmath", + "amsmath", + "deflist", + "fieldlist", + "html_admonition", + "html_image", + "colon_fence", + "smartquotes", + "replacements", + "linkify", + "strikethrough", + "substitution", + "tasklist", + "attrs_inline", +] +myst_number_code_blocks = ["typescript"] +myst_heading_anchors = 2 +myst_footnote_transition = True +myst_dmath_double_inline = True + +# -- HTML output ------------------------------------------------- -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# html_theme = "sphinx_book_theme" html_logo = "_static/logo-wide.svg" html_favicon = "_static/logo-square.svg" @@ -76,27 +131,6 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -myst_enable_extensions = [ - "dollarmath", - "amsmath", - "deflist", - "fieldlist", - "html_admonition", - "html_image", - "colon_fence", - "smartquotes", - "replacements", - "linkify", - "strikethrough", - "substitution", - "tasklist", - "attrs_inline", -] -myst_number_code_blocks = ["typescript"] -myst_heading_anchors = 2 -myst_footnote_transition = True -myst_dmath_double_inline = True - rediraffe_redirects = { "using/intro.md": "sphinx/intro.md", "sphinx/intro.md": "intro.md", @@ -112,47 +146,28 @@ "explain/index.md": "develop/background.md", } -suppress_warnings = ["myst.strikethrough"] +# -- LaTeX output ------------------------------------------------- +latex_engine = "xelatex" -intersphinx_mapping = { - "python": ("https://docs.python.org/3.7", None), - "sphinx": ("https://www.sphinx-doc.org/en/master", None), - "markdown_it": ("https://markdown-it-py.readthedocs.io/en/latest", None), -} +# -- Local Sphinx extensions ------------------------------------------------- -# autodoc_default_options = { -# "show-inheritance": True, -# "special-members": "__init__, __enter__, __exit__", -# "members": True, -# # 'exclude-members': '', -# "undoc-members": True, -# # 'inherited-members': True -# } -autodoc_member_order = "bysource" -nitpicky = True -nitpick_ignore = [ - ("py:class", "docutils.nodes.document"), - ("py:class", "docutils.nodes.docinfo"), - ("py:class", "docutils.nodes.Element"), - ("py:class", "docutils.nodes.Node"), - ("py:class", "docutils.nodes.field_list"), - ("py:class", "docutils.nodes.problematic"), - ("py:class", "docutils.nodes.pending"), - ("py:class", "docutils.nodes.system_message"), - ("py:class", "docutils.statemachine.StringList"), - ("py:class", "docutils.parsers.rst.directives.misc.Include"), - ("py:class", "docutils.parsers.rst.Parser"), - ("py:class", "docutils.utils.Reporter"), - ("py:class", "nodes.Element"), - ("py:class", "nodes.Node"), - ("py:class", "nodes.system_message"), - ("py:class", "Directive"), - ("py:class", "Include"), - ("py:class", "StringList"), - ("py:class", "DocutilsRenderer"), - ("py:class", "MockStateMachine"), -] + +class StripUnsupportedLatex(SphinxPostTransform): + """Remove unsupported nodes from the doctree.""" + + default_priority = 900 + + def run(self): + if not self.app.builder.format == "latex": + return + from docutils import nodes + + for node in self.document.findall(): + if node.tagname == "image" and node["uri"].endswith(".svg"): + node.parent.replace(node, nodes.inline("", "Removed SVG image")) + if node.tagname == "mermaid": + node.parent.replace(node, nodes.inline("", "Removed Mermaid diagram")) def setup(app: Sphinx): @@ -169,3 +184,4 @@ def setup(app: Sphinx): app.add_directive("docutils-cli-help", DocutilsCliHelpDirective) app.add_directive("doc-directive", DirectiveDoc) app.add_directive("myst-warnings", MystWarningsDirective) + app.add_post_transform(StripUnsupportedLatex) diff --git a/docs/syntax/optional.md b/docs/syntax/optional.md index c497d929..8e12e88a 100644 --- a/docs/syntax/optional.md +++ b/docs/syntax/optional.md @@ -138,30 +138,24 @@ For example: ```latex $$ - \begin{eqnarray} - y & = & ax^2 + bx + c \\ - f(x) & = & x^2 + 2xy + y^2 - \end{eqnarray} + y & = ax^2 + bx + c \\ + f(x) & = x^2 + 2xy + y^2 $$ ``` becomes $$ - \begin{eqnarray} - y & = & ax^2 + bx + c \\ - f(x) & = & x^2 + 2xy + y^2 - \end{eqnarray} + y & = ax^2 + bx + c \\ + f(x) & = x^2 + 2xy + y^2 $$ This is equivalent to the following directive: ````md ```{math} - \begin{eqnarray} - y & = & ax^2 + bx + c \\ - f(x) & = & x^2 + 2xy + y^2 - \end{eqnarray} + y & = ax^2 + bx + c \\ + f(x) & = x^2 + 2xy + y^2 ``` ```` diff --git a/tox.ini b/tox.ini index 17a3e1c7..aaa18035 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ whitelist_externals = rm echo commands = - clean: rm -rf docs/_build + clean: rm -rf docs/_build/{posargs:html} sphinx-build -nW --keep-going -b {posargs:html} docs/ docs/_build/{posargs:html} commands_post = echo "open file://{toxinidir}/docs/_build/{posargs:html}/index.html" From 936cba79e873ee8041e45853ede1f33ed5a24787 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Fri, 6 Jan 2023 03:24:40 +0100 Subject: [PATCH 08/20] =?UTF-8?q?=F0=9F=91=8C=20Reference=20attributes=20t?= =?UTF-8?q?itle=20->=20reftitle=20(#666)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sphinx will output reftitle as the HTML title attr. See: https://github.com/sphinx-doc/sphinx/blob/6e6b9ad45194177811551c77f5f4040213d1c661/sphinx/writers/html5.py#L227 --- docs/syntax/optional.md | 4 ++-- myst_parser/mdit_to_docutils/base.py | 21 +++++++++++++++---- .../fixtures/docutil_syntax_elements.md | 4 ++-- .../fixtures/sphinx_syntax_elements.md | 4 ++-- .../test_sphinx_builds/test_basic.html | 2 +- .../test_basic.resolved.xml | 2 +- .../test_sphinx_builds/test_basic.xml | 2 +- 7 files changed, 26 insertions(+), 13 deletions(-) diff --git a/docs/syntax/optional.md b/docs/syntax/optional.md index 8e12e88a..c10c7406 100644 --- a/docs/syntax/optional.md +++ b/docs/syntax/optional.md @@ -760,7 +760,7 @@ For example, the following Markdown: - `A literal with attributes`{#literalid .bg-warning}, {ref}`a reference to the literal <literalid> -- An autolink with attributes: <https://example.com>{.bg-warning} +- An autolink with attributes: <https://example.com>{.bg-warning title="a title"} - [A link with attributes](syntax/attributes){#linkid .bg-warning}, {ref}`a reference to the link <linkid>` @@ -778,7 +778,7 @@ will be parsed as: - `A literal with attributes`{#literalid .bg-warning}, {ref}`a reference to the literal <literalid>` -- An autolink with attributes: <https://example.com>{.bg-warning} +- An autolink with attributes: <https://example.com>{.bg-warning title="a title"} - [A link with attributes](syntax/attributes){#linkid .bg-warning}, {ref}`a reference to the link <linkid>` diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 6b198ea9..4e4d4121 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -379,7 +379,14 @@ def copy_attributes( converters: dict[str, Callable[[str], Any]] | None = None, aliases: dict[str, str] | None = None, ) -> None: - """Copy attributes on the token to the docutils node.""" + """Copy attributes on the token to the docutils node. + + :param token: the token to copy attributes from + :param node: the node to copy attributes to + :param keys: the keys to copy from the token (after aliasing) + :param converters: a dictionary of converters for the attributes + :param aliases: a dictionary mapping the token key name to the node key name + """ if converters is None: converters = {} if aliases is None: @@ -751,7 +758,9 @@ def render_external_url(self, token: SyntaxTreeNode) -> None: """ ref_node = nodes.reference() self.add_line_and_source_path(ref_node, token) - self.copy_attributes(token, ref_node, ("class", "id", "title")) + self.copy_attributes( + token, ref_node, ("class", "id", "reftitle"), aliases={"title": "reftitle"} + ) ref_node["refuri"] = cast(str, token.attrGet("href") or "") with self.current_node_context(ref_node, append=True): self.render_children(token) @@ -769,7 +778,9 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: """ ref_node = nodes.reference() self.add_line_and_source_path(ref_node, token) - self.copy_attributes(token, ref_node, ("class", "id", "title")) + self.copy_attributes( + token, ref_node, ("class", "id", "reftitle"), aliases={"title": "reftitle"} + ) ref_node["refname"] = cast(str, token.attrGet("href") or "") self.document.note_refname(ref_node) with self.current_node_context(ref_node, append=True): @@ -778,7 +789,9 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: def render_autolink(self, token: SyntaxTreeNode) -> None: refuri = escapeHtml(token.attrGet("href") or "") # type: ignore[arg-type] ref_node = nodes.reference() - self.copy_attributes(token, ref_node, ("class", "id")) + self.copy_attributes( + token, ref_node, ("class", "id", "reftitle"), aliases={"title": "reftitle"} + ) ref_node["refuri"] = refuri self.add_line_and_source_path(ref_node, token) with self.current_node_context(ref_node, append=True): diff --git a/tests/test_renderers/fixtures/docutil_syntax_elements.md b/tests/test_renderers/fixtures/docutil_syntax_elements.md index 9b59f3a1..91123a50 100644 --- a/tests/test_renderers/fixtures/docutil_syntax_elements.md +++ b/tests/test_renderers/fixtures/docutil_syntax_elements.md @@ -376,7 +376,7 @@ Link Reference: . <document source="notset"> <paragraph> - <reference refuri="https://www.google.com" title="a title"> + <reference reftitle="a title" refuri="https://www.google.com"> name . @@ -388,7 +388,7 @@ Link Reference short version: . <document source="notset"> <paragraph> - <reference refuri="https://www.google.com" title="a title"> + <reference reftitle="a title" refuri="https://www.google.com"> name . diff --git a/tests/test_renderers/fixtures/sphinx_syntax_elements.md b/tests/test_renderers/fixtures/sphinx_syntax_elements.md index 1ac085c2..2d40bdb5 100644 --- a/tests/test_renderers/fixtures/sphinx_syntax_elements.md +++ b/tests/test_renderers/fixtures/sphinx_syntax_elements.md @@ -379,7 +379,7 @@ Link Reference: . <document source="<src>/index.md"> <paragraph> - <reference refuri="https://www.google.com" title="a title"> + <reference reftitle="a title" refuri="https://www.google.com"> name . @@ -391,7 +391,7 @@ Link Reference short version: . <document source="<src>/index.md"> <paragraph> - <reference refuri="https://www.google.com" title="a title"> + <reference reftitle="a title" refuri="https://www.google.com"> name . diff --git a/tests/test_sphinx/test_sphinx_builds/test_basic.html b/tests/test_sphinx/test_sphinx_builds/test_basic.html index 9813b705..b3e99c79 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_basic.html +++ b/tests/test_sphinx/test_sphinx_builds/test_basic.html @@ -229,7 +229,7 @@ <h1> </a> </p> <p> - <a class="reference external" href="https://www.google.com"> + <a class="reference external" href="https://www.google.com" title="a title"> name </a> </p> diff --git a/tests/test_sphinx/test_sphinx_builds/test_basic.resolved.xml b/tests/test_sphinx/test_sphinx_builds/test_basic.resolved.xml index 8a12765e..18605dc4 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_basic.resolved.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_basic.resolved.xml @@ -140,7 +140,7 @@ <comment classes="block_break" xml:space="preserve"> a block break <paragraph> - <reference refuri="https://www.google.com" title="a title"> + <reference reftitle="a title" refuri="https://www.google.com"> name <literal_block language="default" linenos="False" xml:space="preserve"> def func(a, b=1): diff --git a/tests/test_sphinx/test_sphinx_builds/test_basic.xml b/tests/test_sphinx/test_sphinx_builds/test_basic.xml index 34b0e3c3..a3a1b65d 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_basic.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_basic.xml @@ -141,7 +141,7 @@ <comment classes="block_break" xml:space="preserve"> a block break <paragraph> - <reference refuri="https://www.google.com" title="a title"> + <reference reftitle="a title" refuri="https://www.google.com"> name <literal_block language="default" xml:space="preserve"> def func(a, b=1): From 2a5bd6178e3ba578c94f34804ff852782cc9f85d Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Fri, 6 Jan 2023 04:23:06 +0100 Subject: [PATCH 09/20] =?UTF-8?q?=F0=9F=94=A7=20Rename=20doc=5Fenv=20->=20?= =?UTF-8?q?sphinx=5Fenv=20(#667)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- myst_parser/mdit_to_docutils/sphinx_.py | 32 +++++++++++++------------ 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/myst_parser/mdit_to_docutils/sphinx_.py b/myst_parser/mdit_to_docutils/sphinx_.py index b6607f30..3fed1224 100644 --- a/myst_parser/mdit_to_docutils/sphinx_.py +++ b/myst_parser/mdit_to_docutils/sphinx_.py @@ -30,7 +30,7 @@ class SphinxRenderer(DocutilsRenderer): """ @property - def doc_env(self) -> BuildEnvironment: + def sphinx_env(self) -> BuildEnvironment: return self.document.settings.env def render_internal_link(self, token: SyntaxTreeNode) -> None: @@ -49,8 +49,8 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: ) potential_path = ( - Path(self.doc_env.doc2path(self.doc_env.docname)).parent / destination - if self.doc_env.srcdir # not set in some test situations + Path(self.sphinx_env.doc2path(self.sphinx_env.docname)).parent / destination + if self.sphinx_env.srcdir # not set in some test situations else None ) if ( @@ -58,11 +58,11 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: and potential_path.is_file() and not any( destination.endswith(suffix) - for suffix in self.doc_env.config.source_suffix + for suffix in self.sphinx_env.config.source_suffix ) ): wrap_node = addnodes.download_reference( - refdoc=self.doc_env.docname, + refdoc=self.sphinx_env.docname, reftarget=destination, reftype="myst", refdomain=None, # Added to enable cross-linking @@ -73,7 +73,7 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: text = destination if not token.children else "" else: wrap_node = addnodes.pending_xref( - refdoc=self.doc_env.docname, + refdoc=self.sphinx_env.docname, reftarget=destination, reftype="myst", refdomain=None, # Added to enable cross-linking @@ -110,26 +110,28 @@ def render_heading(self, token: SyntaxTreeNode) -> None: return section = self.current_node - doc_slug = self.doc_env.doc2path(self.doc_env.docname, base=False) + "#" + slug + doc_slug = ( + self.sphinx_env.doc2path(self.sphinx_env.docname, base=False) + "#" + slug + ) # save the reference in the standard domain, so that it can be handled properly - domain = cast(StandardDomain, self.doc_env.get_domain("std")) + domain = cast(StandardDomain, self.sphinx_env.get_domain("std")) if doc_slug in domain.labels: - other_doc = self.doc_env.doc2path(domain.labels[doc_slug][0]) + other_doc = self.sphinx_env.doc2path(domain.labels[doc_slug][0]) self.create_warning( f"duplicate label {doc_slug}, other instance in {other_doc}", MystWarnings.ANCHOR_DUPE, line=section.line, ) labelid = section["ids"][0] - domain.anonlabels[doc_slug] = self.doc_env.docname, labelid + domain.anonlabels[doc_slug] = self.sphinx_env.docname, labelid domain.labels[doc_slug] = ( - self.doc_env.docname, + self.sphinx_env.docname, labelid, clean_astext(section[0]), ) - self.doc_env.metadata[self.doc_env.docname]["myst_anchors"] = True + self.sphinx_env.metadata[self.sphinx_env.docname]["myst_anchors"] = True section["myst-anchor"] = doc_slug def render_math_block_label(self, token: SyntaxTreeNode) -> None: @@ -180,10 +182,10 @@ def add_math_target(self, node: nodes.math_block) -> nodes.target: # Code mainly copied from sphinx.directives.patches.MathDirective # register label to domain - domain = cast(MathDomain, self.doc_env.get_domain("math")) - domain.note_equation(self.doc_env.docname, node["label"], location=node) + domain = cast(MathDomain, self.sphinx_env.get_domain("math")) + domain.note_equation(self.sphinx_env.docname, node["label"], location=node) node["number"] = domain.get_equation_number_for(node["label"]) - node["docname"] = self.doc_env.docname + node["docname"] = self.sphinx_env.docname # create target node node_id = nodes.make_id("equation-%s" % node["label"]) From 74b21f7e7f4c7f8e080e44bf495280b8e01ed520 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Fri, 6 Jan 2023 04:56:51 +0100 Subject: [PATCH 10/20] =?UTF-8?q?=F0=9F=91=8C=20Add=20`render=5Fmath=5Fblo?= =?UTF-8?q?ck=5Flabel`=20for=20docutils=20(#668)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- myst_parser/mdit_to_docutils/base.py | 14 ++++++++++++-- .../fixtures/docutil_syntax_extensions.txt | 7 +++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 4e4d4121..ddeecdff 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -1051,6 +1051,16 @@ def render_math_block(self, token: SyntaxTreeNode) -> None: self.add_line_and_source_path(node, token) self.current_node.append(node) + def render_math_block_label(self, token: SyntaxTreeNode) -> None: + content = token.content + label = token.info + node = nodes.math_block(content, content, nowrap=False, number=None) + self.add_line_and_source_path(node, token) + name = nodes.fully_normalize_name(label) + node["names"].append(name) + self.document.note_explicit_target(node, node) + self.current_node.append(node) + def render_amsmath(self, token: SyntaxTreeNode) -> None: # note docutils does not currently support the nowrap attribute # or equation numbering, so this is overridden in the sphinx renderer @@ -1125,9 +1135,9 @@ def render_myst_role(self, token: SyntaxTreeNode) -> None: ) inliner = MockInliner(self) if role_func: - nodes, messages2 = role_func(name, rawsource, text, lineno, inliner) + _nodes, messages2 = role_func(name, rawsource, text, lineno, inliner) # return nodes, messages + messages2 - self.current_node += nodes + self.current_node += _nodes else: message = self.reporter.error( f'Unknown interpreted text role "{name}".', line=lineno diff --git a/tests/test_renderers/fixtures/docutil_syntax_extensions.txt b/tests/test_renderers/fixtures/docutil_syntax_extensions.txt index 5efcb68a..2dc0d439 100644 --- a/tests/test_renderers/fixtures/docutil_syntax_extensions.txt +++ b/tests/test_renderers/fixtures/docutil_syntax_extensions.txt @@ -10,6 +10,10 @@ $$foo$$ $$ a = 1 $$ + +$$ +b = 2 +$$ (label) . <document source="<string>"> <paragraph> @@ -26,6 +30,9 @@ $$ <math_block nowrap="False" number="True" xml:space="preserve"> a = 1 + <math_block ids="label" names="label" nowrap="False" number="True" xml:space="preserve"> + + b = 2 . [amsmath] --myst-enable-extensions=amsmath From ebb5b9f4e54263d8d62e6dc7ded7f08f4d842e47 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Fri, 6 Jan 2023 18:52:32 +0100 Subject: [PATCH 11/20] =?UTF-8?q?=F0=9F=93=9A=20Minor=20improvements=20to?= =?UTF-8?q?=20docs=20(#670)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/docutils.md | 4 +++ docs/faq/index.md | 4 +-- docs/intro.md | 2 +- docs/syntax/optional.md | 31 ++++++++++--------- docs/syntax/roles-and-directives.md | 7 ++++- docs/syntax/syntax.md | 7 +++-- myst_parser/inventory.py | 15 +++++++++ tests/test_inventory.py | 4 ++- .../test_inv_cli_v2_options4_.yaml | 12 +++++++ 9 files changed, 63 insertions(+), 23 deletions(-) create mode 100644 tests/test_inventory/test_inv_cli_v2_options4_.yaml diff --git a/docs/docutils.md b/docs/docutils.md index b4d04800..ccc47d8c 100644 --- a/docs/docutils.md +++ b/docs/docutils.md @@ -35,6 +35,10 @@ The commands are based on the [Docutils Front-End Tools](https://docutils.source ``` ::: +:::{versionadded} 0.19.0 +`myst-suppress-warnings` replicates the functionality of sphinx's [`suppress_warnings`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-suppress_warnings) for `myst.` warnings in the `docutils` CLI. +::: + The CLI commands can also utilise the [`docutils.conf` configuration file](https://docutils.sourceforge.io/docs/user/config.html) to configure the behaviour of the CLI commands. For example: ``` diff --git a/docs/faq/index.md b/docs/faq/index.md index a62f861d..3f0c27c8 100644 --- a/docs/faq/index.md +++ b/docs/faq/index.md @@ -142,7 +142,7 @@ See the [](syntax/header-anchors) section of extended syntaxes. ::: If you'd like to *automatically* generate targets for each of your section headers, -check out the [`autosectionlabel`](https://www.sphinx-doc.org/en/master/usage/extensions/autosectionlabel.html) +check out the {external+sphinx:std:doc}`autosectionlabel <usage/extensions/autosectionlabel>` sphinx feature. You can activate it in your Sphinx site by adding the following to your `conf.py` file: @@ -179,7 +179,7 @@ Moved to [](myst-warnings) ### Sphinx-specific page front matter Sphinx intercepts front matter and stores them within the global environment -(as discussed [in the deflists documentation](https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html)). +(as discussed in the {external+sphinx:std:doc}`sphinx documentation <usage/restructuredtext/field-lists>`. There are certain front-matter keys (or their translations) that are also recognised specifically by docutils and parsed to inline Markdown: - `author` diff --git a/docs/intro.md b/docs/intro.md index dc657284..f34725fb 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -28,7 +28,7 @@ conda install -c conda-forge myst-parser (intro/sphinx)= ## Enable MyST in Sphinx -To get started with Sphinx, see their [Quickstart Guide](https://www.sphinx-doc.org/en/master/usage/quickstart.html). +To get started with Sphinx, see their {external+sphinx:std:doc}`quick-start guide <usage/quickstart>`. To use the MyST parser in Sphinx, simply add the following to your `conf.py` file: diff --git a/docs/syntax/optional.md b/docs/syntax/optional.md index c10c7406..4ad5073a 100644 --- a/docs/syntax/optional.md +++ b/docs/syntax/optional.md @@ -13,6 +13,7 @@ myst: :width: 200px ``` key4: example + confpy: sphinx `conf.py` {external+sphinx:std:doc}`configuration file <usage/configuration>` --- (syntax/extensions)= @@ -53,12 +54,12 @@ myst_enable_extensions = [ ## Typography -Adding `"smartquotes"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)) will automatically convert standard quotations to their opening/closing variants: +Adding `"smartquotes"` to `myst_enable_extensions` (in the {{ confpy }}) will automatically convert standard quotations to their opening/closing variants: - `'single quotes'`: 'single quotes' - `"double quotes"`: "double quotes" -Adding `"replacements"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)) will automatically convert some common typographic texts +Adding `"replacements"` to `myst_enable_extensions` (in the {{ confpy }}) will automatically convert some common typographic texts text | converted ----- | ---------- @@ -95,7 +96,7 @@ and you will need to suppress the `myst.strikethrough` warning (syntax/math)= ## Math shortcuts -Math is parsed by adding to the `myst_enable_extensions` list option, in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html) one or both of: +Math is parsed by adding to the `myst_enable_extensions` list option, in the {{ confpy }} one or both of: - `"dollarmath"` for parsing of dollar `$` and `$$` encapsulated math. - `"amsmath"` for direct parsing of [amsmath LaTeX environments](https://ctan.org/pkg/amsmath). @@ -230,7 +231,7 @@ See [the extended syntax option](syntax/amsmath). (syntax/mathjax)= ### Mathjax and math parsing -When building HTML using the [sphinx.ext.mathjax](https://www.sphinx-doc.org/en/master/usage/extensions/math.html#module-sphinx.ext.mathjax) extension (enabled by default), +When building HTML using the {external+sphinx:mod}`sphinx.ext.mathjax <sphinx.ext.mathjax>` extension (enabled by default), If `dollarmath` is enabled, Myst-Parser injects the `tex2jax_ignore` (MathJax v2) and `mathjax_ignore` (MathJax v3) classes in to the top-level section of each MyST document, and adds the following default MathJax configuration: MathJax version 2 (see [the tex2jax preprocessor](https://docs.mathjax.org/en/v2.7-latest/options/preprocessors/tex2jax.html#configure-tex2jax): @@ -252,7 +253,7 @@ To change this behaviour, set a custom regex, for identifying HTML classes to pr (syntax/linkify)= ## Linkify -Adding `"linkify"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)) will automatically identify "bare" web URLs and add hyperlinks: +Adding `"linkify"` to `myst_enable_extensions` (in the {{ confpy }}) will automatically identify "bare" web URLs and add hyperlinks: `www.example.com` -> www.example.com @@ -267,7 +268,7 @@ Either directly; `pip install linkify-it-py` or *via* `pip install myst-parser[l ## Substitutions (with Jinja2) -Adding `"substitution"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)) will allow you to add substitutions, added in either the `conf.py` using `myst_substitutions`: +Adding `"substitution"` to `myst_enable_extensions` (in the {{ confpy }}) will allow you to add substitutions, added in either the `conf.py` using `myst_substitutions`: ```python myst_substitutions = { @@ -352,7 +353,7 @@ This may lead to unexpected outcomes. ::: -Substitution references are assessed as [Jinja2 expressions](http://jinja.palletsprojects.com) which can use [filters](https://jinja.palletsprojects.com/en/2.11.x/templates/#list-of-builtin-filters), and also contains the [Sphinx Environment](https://www.sphinx-doc.org/en/master/extdev/envapi.html) in the context (as `env`). +Substitution references are assessed as [Jinja2 expressions](http://jinja.palletsprojects.com) which can use [filters](https://jinja.palletsprojects.com/en/2.11.x/templates/#list-of-builtin-filters), and also contains the {external+sphinx:std:doc}`Sphinx Environment <extdev/envapi>` in the context (as `env`). Therefore you can do things like: ```md @@ -400,7 +401,7 @@ However, since Jinja2 substitutions allow for Python methods to be used, you can ## Code fences using colons -By adding `"colon_fence"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), +By adding `"colon_fence"` to `myst_enable_extensions` (in the {{ confpy }}), you can also use `:::` delimiters to denote code fences, instead of ```` ``` ````. Using colons instead of back-ticks has the benefit of allowing the content to be rendered correctly, when you are working in any standard Markdown editor. @@ -537,7 +538,7 @@ $ myst-anchors -l 2 docs/syntax/optional.md ## Definition Lists -By adding `"deflist"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), +By adding `"deflist"` to `myst_enable_extensions` (in the {{ confpy }}), you will be able to utilise definition lists. Definition lists utilise the [markdown-it-py deflist plugin](markdown_it:md/plugins), which itself is based on the [Pandoc definition list specification](http://johnmacfarlane.net/pandoc/README.html#definition-lists). @@ -616,7 +617,7 @@ Term 3 (syntax/tasklists)= ## Task Lists -By adding `"tasklist"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), +By adding `"tasklist"` to `myst_enable_extensions` (in the {{ confpy }}), you will be able to utilise task lists. Task lists utilise the [markdown-it-py tasklists plugin](markdown_it:md/plugins), and are applied to markdown list items starting with `[ ]` or `[x]`: @@ -725,7 +726,7 @@ Currently `sphinx.ext.autodoc` does not support MyST, see [](howto/autodoc). (syntax/attributes)= ## Inline attributes -By adding `"attrs_inline"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), +By adding `"attrs_inline"` to `myst_enable_extensions` (in the {{ confpy }}), you can enable parsing of inline attributes after certain inline syntaxes. This is adapted from [djot inline attributes](https://htmlpreview.github.io/?https://github.com/jgm/djot/blob/master/doc/syntax.html#inline-attributes), and also related to [pandoc bracketed spans](https://pandoc.org/MANUAL.html#extension-bracketed_spans). @@ -844,7 +845,7 @@ This is usually a bad option, because the HTML is treated as raw text during the HTML parsing to the rescue! -By adding `"html_image"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), +By adding `"html_image"` to `myst_enable_extensions` (in the {{ confpy }}), MySt-Parser will attempt to convert any isolated `img` tags (i.e. not wrapped in any other HTML) to the internal representation used in sphinx. ```html @@ -866,7 +867,7 @@ I'm an inline image: <img src="img/fun-fish.png" height="20px"> ## Markdown Figures -By adding `"colon_fence"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), +By adding `"colon_fence"` to `myst_enable_extensions` (in the {{ confpy }}), we can combine the above two extended syntaxes, to create a fully Markdown compliant version of the `figure` directive named `figure-md`. @@ -908,7 +909,7 @@ As we see here, the target we set can be referenced: ## HTML Admonitions -By adding `"html_admonition"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), +By adding `"html_admonition"` to `myst_enable_extensions` (in the {{ confpy }}), you can enable parsing of `<div class="admonition">` HTML blocks. These blocks will be converted internally to Sphinx admonition directives, and so will work correctly for all output formats. This is helpful when you care about viewing the "source" Markdown, such as in Jupyter Notebooks. @@ -974,7 +975,7 @@ You can also nest HTML admonitions: ## Direct LaTeX Math -By adding `"amsmath"` to `myst_enable_extensions` (in the sphinx `conf.py` [configuration file](https://www.sphinx-doc.org/en/master/usage/configuration.html)), +By adding `"amsmath"` to `myst_enable_extensions` (in the {{ confpy }}), you can enable direct parsing of [amsmath](https://ctan.org/pkg/amsmath) LaTeX equations. These top-level math environments will then be directly parsed: diff --git a/docs/syntax/roles-and-directives.md b/docs/syntax/roles-and-directives.md index 4b3d80a4..3017b527 100644 --- a/docs/syntax/roles-and-directives.md +++ b/docs/syntax/roles-and-directives.md @@ -5,7 +5,12 @@ Roles and directives provide a way to extend the syntax of MyST in an unbound manner, by interpreting a chuck of text as a specific type of markup, according to its name. -Mostly all [docutils roles](https://docutils.sourceforge.io/docs/ref/rst/roles.html), [docutils directives](https://docutils.sourceforge.io/docs/ref/rst/directives.html), [sphinx roles](https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html), or [sphinx directives](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html) can be used in MyST. +Mostly all +[docutils roles](https://docutils.sourceforge.io/docs/ref/rst/roles.html), +[docutils directives](https://docutils.sourceforge.io/docs/ref/rst/directives.html), +{external+sphinx:std:doc}`Sphinx roles <usage/restructuredtext/roles>`, or +{external+sphinx:std:doc}`Sphinx directives <usage/restructuredtext/directives>` +can be used in MyST. ## Syntax diff --git a/docs/syntax/syntax.md b/docs/syntax/syntax.md index 31c7f8d3..0062a551 100644 --- a/docs/syntax/syntax.md +++ b/docs/syntax/syntax.md @@ -85,7 +85,7 @@ would be equivalent to: ### Setting HTML Metadata The front-matter can contain the special key `html_meta`; a dict with data to add to the generated HTML as [`<meta>` elements](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta). -This is equivalent to using the [RST `meta` directive](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#html-metadata). +This is equivalent to using the {external+sphinx:ref}`meta directive <html-meta>`. HTML metadata can also be added globally in the `conf.py` *via* the `myst_html_meta` variable, in which case it will be added to all MyST documents. For each document, the `myst_html_meta` dict will be updated by the document level front-matter `html_meta`, with the front-matter taking precedence. @@ -257,7 +257,8 @@ Target headers are defined with this syntax: (header_target)= ``` -They can then be referred to with the [ref inline role](https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref): +They can then be referred to with the +{external+sphinx:ref}`ref inline role <ref-role>`: ```md {ref}`header_target` @@ -277,7 +278,7 @@ Alternatively using the markdown syntax: [my text](header_target) ``` -is equivalent to using the [any inline role](https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-any): +is equivalent to using the {external+sphinx:ref}`any inline role <any-role>`: ```md {any}`my text <header_target>` diff --git a/myst_parser/inventory.py b/myst_parser/inventory.py index 80082199..da3bf599 100644 --- a/myst_parser/inventory.py +++ b/myst_parser/inventory.py @@ -320,6 +320,12 @@ def inventory_cli(inputs: None | list[str] = None): metavar="NAME", help="Filter the inventory by reference name pattern", ) + parser.add_argument( + "-l", + "--loc", + metavar="LOC", + help="Filter the inventory by reference location pattern", + ) parser.add_argument( "-f", "--format", @@ -363,6 +369,15 @@ def inventory_cli(inputs: None | list[str] = None): if fnmatchcase(n, args.name) } + if args.loc: + for domain in invdata["objects"]: + for otype in list(invdata["objects"][domain]): + invdata["objects"][domain][otype] = { + n: i + for n, i in invdata["objects"][domain][otype].items() + if fnmatchcase(i["loc"], args.loc) + } + # clean up empty items for domain in list(invdata["objects"]): for otype in list(invdata["objects"][domain]): diff --git a/tests/test_inventory.py b/tests/test_inventory.py index f6d1b3d2..80ba7104 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -37,7 +37,9 @@ def test_inv_filter_fnmatch(data_regression): data_regression.check(output) -@pytest.mark.parametrize("options", [(), ("-d", "std"), ("-o", "doc"), ("-n", "ref")]) +@pytest.mark.parametrize( + "options", [(), ("-d", "std"), ("-o", "doc"), ("-n", "ref"), ("-l", "index.html*")] +) def test_inv_cli_v2(options, capsys, file_regression): inventory_cli([str(STATIC / "objects_v2.inv"), "-f", "yaml", *options]) text = capsys.readouterr().out.strip() + "\n" diff --git a/tests/test_inventory/test_inv_cli_v2_options4_.yaml b/tests/test_inventory/test_inv_cli_v2_options4_.yaml new file mode 100644 index 00000000..797d21b1 --- /dev/null +++ b/tests/test_inventory/test_inv_cli_v2_options4_.yaml @@ -0,0 +1,12 @@ +name: Python +version: '' +objects: + std: + label: + ref: + loc: index.html#ref + text: Title + doc: + index: + loc: index.html + text: Title From 66881efc16492e2291d36539d3484c92c454612a Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 9 Jan 2023 00:32:53 +0100 Subject: [PATCH 12/20] =?UTF-8?q?=F0=9F=91=8C=E2=80=BC=EF=B8=8F=20Allow=20?= =?UTF-8?q?meta=5Fhtml/substitutions=20in=20docutils=20(#672)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactors `myst_parser/parsers/docutils_.py` slightly, by removing `DOCUTILS_EXCLUDED_ARGS`, and removing the `excluded` argument from `create_myst_settings_spec` and `create_myst_config`. --- .github/workflows/tests.yml | 4 +- docs/docutils.md | 3 + myst_parser/_docs.py | 2 +- myst_parser/config/dc_validators.py | 2 +- myst_parser/config/main.py | 20 +++-- myst_parser/mdit_to_docutils/base.py | 14 ++-- myst_parser/parsers/docutils_.py | 79 +++++++++++-------- pyproject.toml | 5 ++ tests/test_renderers/fixtures/myst-config.txt | 21 +++++ tests/test_renderers/test_myst_config.py | 7 +- 10 files changed, 101 insertions(+), 56 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6ca1ae05..07aac7b1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,7 +86,7 @@ jobs: run: python .github/workflows/docutils_setup.py pyproject.toml README.md - name: Install dependencies run: | - pip install . pytest~=6.2 pytest-param-files~=0.3.3 pygments docutils==${{ matrix.docutils-version }} + pip install .[linkify,testing-docutils] docutils==${{ matrix.docutils-version }} - name: ensure sphinx is not installed run: | python -c "\ @@ -97,7 +97,7 @@ jobs: else: raise AssertionError()" - name: Run pytest for docutils-only tests - run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py tests/test_renderers/test_include_directive.py + run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py tests/test_renderers/test_include_directive.py tests/test_renderers/test_myst_config.py - name: Run docutils CLI run: echo "test" | myst-docutils-html diff --git a/docs/docutils.md b/docs/docutils.md index ccc47d8c..10ea237e 100644 --- a/docs/docutils.md +++ b/docs/docutils.md @@ -46,6 +46,9 @@ The CLI commands can also utilise the [`docutils.conf` configuration file](https [general] myst-enable-extensions: deflist,linkify myst-footnote-transition: no +myst-substitutions: + key1: value1 + key2: value2 # These entries affect specific HTML output: [html writers] diff --git a/myst_parser/_docs.py b/myst_parser/_docs.py index cdef6c46..9fcdf156 100644 --- a/myst_parser/_docs.py +++ b/myst_parser/_docs.py @@ -77,7 +77,7 @@ def run(self): continue # filter by sphinx options - if "sphinx" in self.options and field.metadata.get("sphinx_exclude"): + if "sphinx" in self.options and field.metadata.get("docutils_only"): continue if "extensions" in self.options: diff --git a/myst_parser/config/dc_validators.py b/myst_parser/config/dc_validators.py index 09c94309..e612a8cc 100644 --- a/myst_parser/config/dc_validators.py +++ b/myst_parser/config/dc_validators.py @@ -38,7 +38,7 @@ def validate_fields(inst: Any) -> None: class ValidatorType(Protocol): def __call__( - self, inst: bytes, field: dc.Field, value: Any, suffix: str = "" + self, inst: Any, field: dc.Field, value: Any, suffix: str = "" ) -> None: ... diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py index b32714a5..5ca5ec44 100644 --- a/myst_parser/config/main.py +++ b/myst_parser/config/main.py @@ -26,9 +26,10 @@ ) -def check_extensions(_, __, value): +def check_extensions(_, field: dc.Field, value: Any): + """Check that the extensions are a list of known strings""" if not isinstance(value, Iterable): - raise TypeError(f"'enable_extensions' not iterable: {value}") + raise TypeError(f"'{field.name}' not iterable: {value}") diff = set(value).difference( [ "amsmath", @@ -49,16 +50,17 @@ def check_extensions(_, __, value): ] ) if diff: - raise ValueError(f"'enable_extensions' items not recognised: {diff}") + raise ValueError(f"'{field.name}' items not recognised: {diff}") -def check_sub_delimiters(_, __, value): +def check_sub_delimiters(_, field: dc.Field, value: Any): + """Check that the sub_delimiters are a tuple of length 2 of strings of length 1""" if (not isinstance(value, (tuple, list))) or len(value) != 2: - raise TypeError(f"myst_sub_delimiters is not a tuple of length 2: {value}") + raise TypeError(f"'{field.name}' is not a tuple of length 2: {value}") for delim in value: if (not isinstance(delim, str)) or len(delim) != 1: raise TypeError( - f"myst_sub_delimiters does not contain strings of length 1: {value}" + f"'{field.name}' does not contain strings of length 1: {value}" ) @@ -125,6 +127,7 @@ class MdParserConfig: deep_iterable(instance_of(str), instance_of((list, tuple))) ), "help": "Sphinx domain names to search in for link references", + "sphinx_only": True, }, ) @@ -149,6 +152,7 @@ class MdParserConfig: metadata={ "validator": optional(in_([1, 2, 3, 4, 5, 6, 7])), "help": "Heading level depth to assign HTML anchors", + "sphinx_only": True, }, ) @@ -158,6 +162,7 @@ class MdParserConfig: "validator": optional(is_callable), "help": "Function for creating heading anchors", "global_only": True, + "sphinx_only": True, }, ) @@ -210,6 +215,7 @@ class MdParserConfig: "validator": check_sub_delimiters, "help": "Substitution delimiters", "extension": "substitutions", + "sphinx_only": True, }, ) @@ -262,6 +268,7 @@ class MdParserConfig: "help": "Update sphinx.ext.mathjax configuration to ignore `$` delimiters", "extension": "dollarmath", "global_only": True, + "sphinx_only": True, }, ) @@ -272,6 +279,7 @@ class MdParserConfig: "help": "MathJax classes to add to math HTML", "extension": "dollarmath", "global_only": True, + "sphinx_only": True, }, ) diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index ddeecdff..2e8096fc 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -1461,16 +1461,12 @@ def html_meta_to_nodes( return [] try: - # if sphinx available - from sphinx.addnodes import meta as meta_cls - except ImportError: - try: - # docutils >= 0.19 - meta_cls = nodes.meta # type: ignore - except AttributeError: - from docutils.parsers.rst.directives.html import MetaBody + meta_cls = nodes.meta + except AttributeError: + # docutils-0.17 or older + from docutils.parsers.rst.directives.html import MetaBody - meta_cls = MetaBody.meta # type: ignore + meta_cls = MetaBody.meta output = [] diff --git a/myst_parser/parsers/docutils_.py b/myst_parser/parsers/docutils_.py index 84f4ff92..efa2e78d 100644 --- a/myst_parser/parsers/docutils_.py +++ b/myst_parser/parsers/docutils_.py @@ -2,6 +2,7 @@ from dataclasses import Field from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union +import yaml from docutils import frontend, nodes from docutils.core import default_description, publish_cmdline from docutils.parsers.rst import Parser as RstParser @@ -58,32 +59,39 @@ def __bool__(self): """Sentinel for arguments not set through docutils.conf.""" -DOCUTILS_EXCLUDED_ARGS = ( - # docutils.conf can't represent callables - "heading_slug_func", - # docutils.conf can't represent dicts - "html_meta", - "substitutions", - # we can't add substitutions so not needed - "sub_delimiters", - # sphinx only options - "heading_anchors", - "ref_domains", - "update_mathjax", - "mathjax_classes", -) -"""Names of settings that cannot be set in docutils.conf.""" +def _create_validate_yaml(field: Field): + """Create a deserializer/validator for a json setting.""" + + def _validate_yaml( + setting, value, option_parser, config_parser=None, config_section=None + ): + """Check/normalize a key-value pair setting. + + Items delimited by `,`, and key-value pairs delimited by `=`. + """ + try: + output = yaml.safe_load(value) + except Exception: + raise ValueError("Invalid YAML string") + if "validator" in field.metadata: + field.metadata["validator"](None, field, output) + return output + + return _validate_yaml def _attr_to_optparse_option(at: Field, default: Any) -> Tuple[dict, str]: - """Convert a field into a Docutils optparse options dict.""" + """Convert a field into a Docutils optparse options dict. + + :returns: (option_dict, default) + """ if at.type is int: - return {"metavar": "<int>", "validator": _validate_int}, f"(default: {default})" + return {"metavar": "<int>", "validator": _validate_int}, str(default) if at.type is bool: return { "metavar": "<boolean>", "validator": frontend.validate_boolean, - }, f"(default: {default})" + }, str(default) if at.type is str: return { "metavar": "<str>", @@ -96,28 +104,32 @@ def _attr_to_optparse_option(at: Field, default: Any) -> Tuple[dict, str]: "metavar": f"<{'|'.join(repr(a) for a in args)}>", "type": "choice", "choices": args, - }, f"(default: {default!r})" + }, repr(default) if at.type in (Iterable[str], Sequence[str]): return { "metavar": "<comma-delimited>", "validator": frontend.validate_comma_separated_list, - }, f"(default: '{','.join(default)}')" + }, ",".join(default) if at.type == Tuple[str, str]: return { "metavar": "<str,str>", "validator": _create_validate_tuple(2), - }, f"(default: '{','.join(default)}')" + }, ",".join(default) if at.type == Union[int, type(None)]: return { "metavar": "<null|int>", "validator": _validate_int, - }, f"(default: {default})" + }, str(default) if at.type == Union[Iterable[str], type(None)]: - default_str = ",".join(default) if default else "" return { "metavar": "<null|comma-delimited>", "validator": frontend.validate_comma_separated_list, - }, f"(default: {default_str!r})" + }, ",".join(default) if default else "" + if get_origin(at.type) is dict: + return { + "metavar": "<yaml-dict>", + "validator": _create_validate_yaml(at), + }, str(default) if default else "" raise AssertionError( f"Configuration option {at.name} not set up for use in docutils.conf." ) @@ -133,34 +145,33 @@ def attr_to_optparse_option( name = f"{prefix}{attribute.name}" flag = "--" + name.replace("_", "-") options = {"dest": name, "default": DOCUTILS_UNSET} - at_options, type_str = _attr_to_optparse_option(attribute, default) + at_options, default_str = _attr_to_optparse_option(attribute, default) options.update(at_options) help_str = attribute.metadata.get("help", "") if attribute.metadata else "" - return (f"{help_str} {type_str}", [flag], options) + if default_str: + help_str += f" (default: {default_str})" + return (help_str, [flag], options) -def create_myst_settings_spec( - excluded: Sequence[str], config_cls=MdParserConfig, prefix: str = "myst_" -): +def create_myst_settings_spec(config_cls=MdParserConfig, prefix: str = "myst_"): """Return a list of Docutils setting for the docutils MyST section.""" defaults = config_cls() return tuple( attr_to_optparse_option(at, getattr(defaults, at.name), prefix) for at in config_cls.get_fields() - if at.name not in excluded + if (not at.metadata.get("sphinx_only", False)) ) def create_myst_config( settings: frontend.Values, - excluded: Sequence[str], config_cls=MdParserConfig, prefix: str = "myst_", ): """Create a configuration instance from the given settings.""" values = {} for attribute in config_cls.get_fields(): - if attribute.name in excluded: + if attribute.metadata.get("sphinx_only", False): continue setting = f"{prefix}{attribute.name}" val = getattr(settings, setting, DOCUTILS_UNSET) @@ -178,7 +189,7 @@ class Parser(RstParser): settings_spec = ( "MyST options", None, - create_myst_settings_spec(DOCUTILS_EXCLUDED_ARGS), + create_myst_settings_spec(), *RstParser.settings_spec, ) """Runtime settings specification.""" @@ -209,7 +220,7 @@ def parse(self, inputstring: str, document: nodes.document) -> None: # create parsing configuration from the global config try: - config = create_myst_config(document.settings, DOCUTILS_EXCLUDED_ARGS) + config = create_myst_config(document.settings) except Exception as exc: error = document.reporter.error(f"Global myst configuration invalid: {exc}") document.append(error) diff --git a/pyproject.toml b/pyproject.toml index b3b3a90e..e370148d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,11 @@ testing = [ "pytest-param-files~=0.3.4", "sphinx-pytest", ] +testing-docutils = [ + "pygments", + "pytest>=6,<7", + "pytest-param-files~=0.3.4", +] [project.scripts] myst-anchors = "myst_parser.cli:print_anchors" diff --git a/tests/test_renderers/fixtures/myst-config.txt b/tests/test_renderers/fixtures/myst-config.txt index 8a71971f..562f99b1 100644 --- a/tests/test_renderers/fixtures/myst-config.txt +++ b/tests/test_renderers/fixtures/myst-config.txt @@ -156,6 +156,27 @@ www.commonmark.org/he<lp <lp . +[html_meta] --myst-html-meta='{"keywords": "Sphinx, MyST"}' +. +text +. +<document source="<string>"> + <meta content="Sphinx, MyST" name="keywords"> + <paragraph> + text +. + +[substitutions] --myst-enable-extensions=substitution --myst-substitutions='{"a": "b", "c": "d"}' +. +{{a}} {{c}} +. +<document source="<string>"> + <paragraph> + b + + d +. + [attrs_inline_span] --myst-enable-extensions=attrs_inline . [content]{#id .a .b} diff --git a/tests/test_renderers/test_myst_config.py b/tests/test_renderers/test_myst_config.py index 31e2444e..0640238a 100644 --- a/tests/test_renderers/test_myst_config.py +++ b/tests/test_renderers/test_myst_config.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -from docutils.core import Publisher, publish_doctree +from docutils.core import Publisher, publish_string from myst_parser.parsers.docutils_ import Parser @@ -25,13 +25,14 @@ def test_cmdline(file_params): f"Failed to parse commandline: {file_params.description}\n{err}" ) report_stream = StringIO() + settings["output_encoding"] = "unicode" settings["warning_stream"] = report_stream - doctree = publish_doctree( + output = publish_string( file_params.content, parser=Parser(), + writer_name="pseudoxml", settings_overrides=settings, ) - output = doctree.pformat() warnings = report_stream.getvalue() if warnings: output += "\n" + warnings From 38559cb0e1f360bb76b3d055587130bc4af4af20 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Sat, 7 Jan 2023 04:03:44 +0100 Subject: [PATCH 13/20] =?UTF-8?q?=F0=9F=91=8C=20Refactor=20inventory=20to?= =?UTF-8?q?=20use=20wildcards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- myst_parser/inventory.py | 312 ++++++++++++------ tests/test_inventory.py | 13 +- tests/test_inventory/test_inv_cli_v1.yaml | 1 + .../test_inv_cli_v2_options0_.yaml | 1 + .../test_inv_cli_v2_options1_.yaml | 1 + .../test_inv_cli_v2_options2_.yaml | 1 + .../test_inv_cli_v2_options3_.yaml | 1 + .../test_inv_cli_v2_options4_.yaml | 1 + tests/test_inventory/test_inv_filter.yml | 7 +- ...match.yml => test_inv_filter_wildcard.yml} | 28 +- 10 files changed, 241 insertions(+), 125 deletions(-) rename tests/test_inventory/{test_inv_filter_fnmatch.yml => test_inv_filter_wildcard.yml} (51%) diff --git a/myst_parser/inventory.py b/myst_parser/inventory.py index da3bf599..b8e298f7 100644 --- a/myst_parser/inventory.py +++ b/myst_parser/inventory.py @@ -8,11 +8,11 @@ from __future__ import annotations import argparse +import functools import json import re import zlib from dataclasses import asdict, dataclass -from fnmatch import fnmatchcase from typing import IO, TYPE_CHECKING, Iterator from urllib.request import urlopen @@ -21,14 +21,16 @@ from ._compat import TypedDict if TYPE_CHECKING: - from sphinx.util.typing import Inventory + # domain_type:object_type -> name -> (project, version, loc, text) + # the `loc` includes the base url, also null `text` is denoted by "-" + from sphinx.util.typing import Inventory as SphinxInventoryType class InventoryItemType(TypedDict): """A single inventory item.""" loc: str - """The relative location of the item.""" + """The location of the item (relative if base_url not None).""" text: str | None """Implicit text to show for the item.""" @@ -40,12 +42,14 @@ class InventoryType(TypedDict): """The name of the project.""" version: str """The version of the project.""" + base_url: str | None + """The base URL of the `loc`s.""" objects: dict[str, dict[str, dict[str, InventoryItemType]]] """Mapping of domain -> object type -> name -> item.""" -def from_sphinx(inv: Inventory) -> InventoryType: - """Convert a Sphinx inventory to one that is JSON compliant.""" +def from_sphinx(inv: SphinxInventoryType) -> InventoryType: + """Convert from a Sphinx compliant format.""" project = "" version = "" objs: dict[str, dict[str, dict[str, InventoryItemType]]] = {} @@ -65,13 +69,14 @@ def from_sphinx(inv: Inventory) -> InventoryType: return { "name": project, "version": version, + "base_url": None, "objects": objs, } -def to_sphinx(inv: InventoryType) -> Inventory: - """Convert a JSON compliant inventory to one that is Sphinx compliant.""" - objs: Inventory = {} +def to_sphinx(inv: InventoryType) -> SphinxInventoryType: + """Convert to a Sphinx compliant format.""" + objs: SphinxInventoryType = {} for domain_name, obj_types in inv["objects"].items(): for obj_type, refs in obj_types.items(): for refname, refdata in refs.items(): @@ -84,25 +89,26 @@ def to_sphinx(inv: InventoryType) -> Inventory: return objs -def load(stream: IO) -> InventoryType: +def load(stream: IO, base_url: str | None = None) -> InventoryType: """Load inventory data from a stream.""" reader = InventoryFileReader(stream) line = reader.readline().rstrip() if line == "# Sphinx inventory version 1": - return _load_v1(reader) + return _load_v1(reader, base_url) elif line == "# Sphinx inventory version 2": - return _load_v2(reader) + return _load_v2(reader, base_url) else: raise ValueError("invalid inventory header: %s" % line) -def _load_v1(stream: InventoryFileReader) -> InventoryType: +def _load_v1(stream: InventoryFileReader, base_url: str | None) -> InventoryType: """Load inventory data (format v1) from a stream.""" projname = stream.readline().rstrip()[11:] version = stream.readline().rstrip()[11:] invdata: InventoryType = { "name": projname, "version": version, + "base_url": base_url, "objects": {}, } for line in stream.readlines(): @@ -120,13 +126,14 @@ def _load_v1(stream: InventoryFileReader) -> InventoryType: return invdata -def _load_v2(stream: InventoryFileReader) -> InventoryType: +def _load_v2(stream: InventoryFileReader, base_url: str | None) -> InventoryType: """Load inventory data (format v2) from a stream.""" projname = stream.readline().rstrip()[11:] version = stream.readline().rstrip()[11:] invdata: InventoryType = { "name": projname, "version": version, + "base_url": base_url, "objects": {}, } line = stream.readline() @@ -225,6 +232,46 @@ def read_compressed_lines(self) -> Iterator[str]: pos = buf.find(b"\n") +@functools.lru_cache(maxsize=256) +def _create_regex(pat: str) -> re.Pattern: + r"""Create a regex from a pattern, that can include `*` wildcards, + to match 0 or more characters. + + `\*` is translated as a literal `*`. + """ + regex = "" + backslash_last = False + for char in pat: + if backslash_last and char == "*": + regex += re.escape(char) + backslash_last = False + continue + if backslash_last: + regex += re.escape("\\") + backslash_last = False + if char == "\\": + backslash_last = True + continue + if char == "*": + regex += ".*" + continue + regex += re.escape(char) + + return re.compile(regex) + + +def match_with_wildcard(name: str, pattern: str | None) -> bool: + r"""Match a whole name with a pattern, that can include `*` wildcards, + to match 0 or more characters. + + To include a literal `*` in the pattern, use `\*`. + """ + if pattern is None: + return True + regex = _create_regex(pattern) + return regex.fullmatch(name) is not None + + @dataclass class InvMatch: """A match from an inventory.""" @@ -233,69 +280,137 @@ class InvMatch: domain: str otype: str name: str - proj: str + project: str version: str - uri: str - text: str + base_url: str | None + loc: str + text: str | None def asdict(self) -> dict[str, str]: return asdict(self) def filter_inventories( - inventories: dict[str, Inventory], - ref_target: str, + inventories: dict[str, InventoryType], *, - ref_inv: None | str = None, - ref_domain: None | str = None, - ref_otype: None | str = None, - fnmatch_target=False, + invs: str | None = None, + domains: str | None = None, + otypes: str | None = None, + targets: str | None = None, ) -> Iterator[InvMatch]: - """Resolve a cross-reference in the loaded sphinx inventories. + r"""Filter a set of inventories. + + Filters are strings that can include `*` wildcards, to match 0 or more characters. + To include a literal `*` in the pattern, use `\*`. :param inventories: Mapping of inventory name to inventory data - :param ref_target: The target to search for - :param ref_inv: The name of the sphinx inventory to search, if None then - all inventories will be searched - :param ref_domain: The name of the domain to search, if None then all domains - will be searched - :param ref_otype: The type of object to search for, if None then all types will be searched - :param fnmatch_target: Whether to match ref_target using fnmatchcase - - :yields: matching results + :param invs: the inventory key filter + :param domains: the domain name filter + :param otypes: the object type filter + :param targets: the target name filter """ for inv_name, inv_data in inventories.items(): - - if ref_inv is not None and ref_inv != inv_name: + if not match_with_wildcard(inv_name, invs): continue + for domain_name, dom_data in inv_data["objects"].items(): + if not match_with_wildcard(domain_name, domains): + continue + for obj_type, obj_data in dom_data.items(): + if not match_with_wildcard(obj_type, otypes): + continue + for target, item_data in obj_data.items(): + if match_with_wildcard(target, targets): + yield InvMatch( + inv=inv_name, + domain=domain_name, + otype=obj_type, + name=target, + project=inv_data["name"], + version=inv_data["version"], + base_url=inv_data["base_url"], + loc=item_data["loc"], + text=item_data["text"], + ) - for domain_obj_name, data in inv_data.items(): +def filter_sphinx_inventories( + inventories: dict[str, SphinxInventoryType], + *, + invs: str | None = None, + domains: str | None = None, + otypes: str | None = None, + targets: str | None = None, +) -> Iterator[InvMatch]: + r"""Filter a set of sphinx style inventories. + + Filters are strings that can include `*` wildcards, to match 0 or more characters. + To include a literal `*` in the pattern, use `\*`. + + :param inventories: Mapping of inventory name to inventory data + :param invs: the inventory key filter + :param domains: the domain name filter + :param otypes: the object type filter + :param targets: the target name filter + """ + for inv_name, inv_data in inventories.items(): + if not match_with_wildcard(inv_name, invs): + continue + for domain_obj_name, data in inv_data.items(): if ":" not in domain_obj_name: continue - domain_name, obj_type = domain_obj_name.split(":", 1) - - if ref_domain is not None and ref_domain != domain_name: + if not ( + match_with_wildcard(domain_name, domains) + and match_with_wildcard(obj_type, otypes) + ): continue + for target in data: + if match_with_wildcard(target, targets): + project, version, loc, text = data[target] + yield ( + InvMatch( + inv=inv_name, + domain=domain_name, + otype=obj_type, + name=target, + project=project, + version=version, + base_url=None, + loc=loc, + text=None if (not text or text == "-") else text, + ) + ) - if ref_otype is not None and ref_otype != obj_type: - continue - if not fnmatch_target and ref_target in data: - yield ( - InvMatch( - inv_name, domain_name, obj_type, ref_target, *data[ref_target] - ) - ) - elif fnmatch_target: - for target in data: - if fnmatchcase(target, ref_target): - yield ( - InvMatch( - inv_name, domain_name, obj_type, target, *data[target] - ) - ) +def filter_string( + invs: str | None, + domains: str | None, + otype: str | None, + target: str | None, + *, + delimiter: str = ":", +) -> str: + """Create a string representation of the filter, from the given arguments.""" + str_items = [] + for item in (invs, domains, otype, target): + if item is None: + str_items.append("*") + elif delimiter in item: + str_items.append(f'"{item}"') + else: + str_items.append(f"{item}") + return delimiter.join(str_items) + + +def fetch_inventory( + uri: str, *, timeout: None | float = None, base_url: None | str = None +) -> InventoryType: + """Fetch an inventory from a URL or local path.""" + if uri.startswith("http://") or uri.startswith("https://"): + with urlopen(uri, timeout=timeout) as stream: + return load(stream, base_url=base_url) + with open(uri, "rb") as stream: + return load(stream, base_url=base_url) def inventory_cli(inputs: None | list[str] = None): @@ -306,90 +421,83 @@ def inventory_cli(inputs: None | list[str] = None): "-d", "--domain", metavar="DOMAIN", - help="Filter the inventory by domain pattern", + default="*", + help="Filter the inventory by domain (`*` = wildcard)", ) parser.add_argument( "-o", "--object-type", metavar="TYPE", - help="Filter the inventory by object type pattern", + default="*", + help="Filter the inventory by object type (`*` = wildcard)", ) parser.add_argument( "-n", "--name", metavar="NAME", - help="Filter the inventory by reference name pattern", + default="*", + help="Filter the inventory by reference name (`*` = wildcard)", ) parser.add_argument( "-l", "--loc", metavar="LOC", - help="Filter the inventory by reference location pattern", + help="Filter the inventory by reference location (`*` = wildcard)", ) parser.add_argument( "-f", "--format", choices=["yaml", "json"], default="yaml", - help="Output format (default: yaml)", + help="Output format", + ) + parser.add_argument( + "--timeout", + type=float, + metavar="SECONDS", + help="Timeout for fetching the inventory", ) args = parser.parse_args(inputs) - if args.uri.startswith("http"): + base_url = None + if args.uri.startswith("http://") or args.uri.startswith("https://"): try: - with urlopen(args.uri) as stream: + with urlopen(args.uri, timeout=args.timeout) as stream: invdata = load(stream) + base_url = args.uri.rsplit("/", 1)[0] except Exception: - with urlopen(args.uri + "/objects.inv") as stream: + with urlopen(args.uri + "/objects.inv", timeout=args.timeout) as stream: invdata = load(stream) + base_url = args.uri else: with open(args.uri, "rb") as stream: invdata = load(stream) - # filter the inventory - if args.domain: - invdata["objects"] = { - d: invdata["objects"][d] - for d in invdata["objects"] - if fnmatchcase(d, args.domain) + filtered: InventoryType = { + "name": invdata["name"], + "version": invdata["version"], + "base_url": base_url, + "objects": {}, + } + for match in filter_inventories( + {"": invdata}, + domains=args.domain, + otypes=args.object_type, + targets=args.name, + ): + if args.loc and not match_with_wildcard(match.loc, args.loc): + continue + filtered["objects"].setdefault(match.domain, {}).setdefault(match.otype, {})[ + match.name + ] = { + "loc": match.loc, + "text": match.text, } - if args.object_type: - for domain in list(invdata["objects"]): - invdata["objects"][domain] = { - t: invdata["objects"][domain][t] - for t in invdata["objects"][domain] - if fnmatchcase(t, args.object_type) - } - if args.name: - for domain in invdata["objects"]: - for otype in list(invdata["objects"][domain]): - invdata["objects"][domain][otype] = { - n: invdata["objects"][domain][otype][n] - for n in invdata["objects"][domain][otype] - if fnmatchcase(n, args.name) - } - - if args.loc: - for domain in invdata["objects"]: - for otype in list(invdata["objects"][domain]): - invdata["objects"][domain][otype] = { - n: i - for n, i in invdata["objects"][domain][otype].items() - if fnmatchcase(i["loc"], args.loc) - } - - # clean up empty items - for domain in list(invdata["objects"]): - for otype in list(invdata["objects"][domain]): - if not invdata["objects"][domain][otype]: - del invdata["objects"][domain][otype] - if not invdata["objects"][domain]: - del invdata["objects"][domain] if args.format == "json": - print(json.dumps(invdata, indent=2, sort_keys=False)) + print(json.dumps(filtered, indent=2, sort_keys=False)) else: - print(yaml.dump(invdata, sort_keys=False)) + print(yaml.dump(filtered, sort_keys=False)) if __name__ == "__main__": diff --git a/tests/test_inventory.py b/tests/test_inventory.py index 80ba7104..795a1144 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -22,18 +22,15 @@ def test_convert_roundtrip(): def test_inv_filter(data_regression): with (STATIC / "objects_v2.inv").open("rb") as f: - inv = to_sphinx(load(f)) - output = [m.asdict() for m in filter_inventories({"inv": inv}, "index")] + inv = load(f) + output = [m.asdict() for m in filter_inventories({"inv": inv}, targets="index")] data_regression.check(output) -def test_inv_filter_fnmatch(data_regression): +def test_inv_filter_wildcard(data_regression): with (STATIC / "objects_v2.inv").open("rb") as f: - inv = to_sphinx(load(f)) - output = [ - m.asdict() - for m in filter_inventories({"inv": inv}, "*index", fnmatch_target=True) - ] + inv = load(f) + output = [m.asdict() for m in filter_inventories({"inv": inv}, targets="*index")] data_regression.check(output) diff --git a/tests/test_inventory/test_inv_cli_v1.yaml b/tests/test_inventory/test_inv_cli_v1.yaml index 1eb78b85..f2d18cb4 100644 --- a/tests/test_inventory/test_inv_cli_v1.yaml +++ b/tests/test_inventory/test_inv_cli_v1.yaml @@ -1,5 +1,6 @@ name: foo version: '1.0' +base_url: null objects: py: module: diff --git a/tests/test_inventory/test_inv_cli_v2_options0_.yaml b/tests/test_inventory/test_inv_cli_v2_options0_.yaml index d51b779f..eafd8659 100644 --- a/tests/test_inventory/test_inv_cli_v2_options0_.yaml +++ b/tests/test_inventory/test_inv_cli_v2_options0_.yaml @@ -1,5 +1,6 @@ name: Python version: '' +base_url: null objects: std: label: diff --git a/tests/test_inventory/test_inv_cli_v2_options1_.yaml b/tests/test_inventory/test_inv_cli_v2_options1_.yaml index d51b779f..eafd8659 100644 --- a/tests/test_inventory/test_inv_cli_v2_options1_.yaml +++ b/tests/test_inventory/test_inv_cli_v2_options1_.yaml @@ -1,5 +1,6 @@ name: Python version: '' +base_url: null objects: std: label: diff --git a/tests/test_inventory/test_inv_cli_v2_options2_.yaml b/tests/test_inventory/test_inv_cli_v2_options2_.yaml index 9ea4200f..4408591d 100644 --- a/tests/test_inventory/test_inv_cli_v2_options2_.yaml +++ b/tests/test_inventory/test_inv_cli_v2_options2_.yaml @@ -1,5 +1,6 @@ name: Python version: '' +base_url: null objects: std: doc: diff --git a/tests/test_inventory/test_inv_cli_v2_options3_.yaml b/tests/test_inventory/test_inv_cli_v2_options3_.yaml index e64e40ee..157c96b3 100644 --- a/tests/test_inventory/test_inv_cli_v2_options3_.yaml +++ b/tests/test_inventory/test_inv_cli_v2_options3_.yaml @@ -1,5 +1,6 @@ name: Python version: '' +base_url: null objects: std: label: diff --git a/tests/test_inventory/test_inv_cli_v2_options4_.yaml b/tests/test_inventory/test_inv_cli_v2_options4_.yaml index 797d21b1..56469cec 100644 --- a/tests/test_inventory/test_inv_cli_v2_options4_.yaml +++ b/tests/test_inventory/test_inv_cli_v2_options4_.yaml @@ -1,5 +1,6 @@ name: Python version: '' +base_url: null objects: std: label: diff --git a/tests/test_inventory/test_inv_filter.yml b/tests/test_inventory/test_inv_filter.yml index 8ac5d5f0..bdee9d7d 100644 --- a/tests/test_inventory/test_inv_filter.yml +++ b/tests/test_inventory/test_inv_filter.yml @@ -1,8 +1,9 @@ -- domain: std +- base_url: null + domain: std inv: inv + loc: index.html name: index otype: doc - proj: Python + project: Python text: Title - uri: index.html version: '' diff --git a/tests/test_inventory/test_inv_filter_fnmatch.yml b/tests/test_inventory/test_inv_filter_wildcard.yml similarity index 51% rename from tests/test_inventory/test_inv_filter_fnmatch.yml rename to tests/test_inventory/test_inv_filter_wildcard.yml index aa24f602..8d972732 100644 --- a/tests/test_inventory/test_inv_filter_fnmatch.yml +++ b/tests/test_inventory/test_inv_filter_wildcard.yml @@ -1,32 +1,36 @@ -- domain: std +- base_url: null + domain: std inv: inv + loc: genindex.html name: genindex otype: label - proj: Python + project: Python text: Index - uri: genindex.html version: '' -- domain: std +- base_url: null + domain: std inv: inv + loc: py-modindex.html name: modindex otype: label - proj: Python + project: Python text: Module Index - uri: py-modindex.html version: '' -- domain: std +- base_url: null + domain: std inv: inv + loc: py-modindex.html name: py-modindex otype: label - proj: Python + project: Python text: Python Module Index - uri: py-modindex.html version: '' -- domain: std +- base_url: null + domain: std inv: inv + loc: index.html name: index otype: doc - proj: Python + project: Python text: Title - uri: index.html version: '' From 4904a4fbca75efb584bfa327595c646bf892cc42 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 9 Jan 2023 20:47:06 +0100 Subject: [PATCH 14/20] =?UTF-8?q?=E2=9C=A8=20NEW:=20Add=20`inv=5Flink`=20e?= =?UTF-8?q?xtension?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 +- docs/conf.py | 1 + docs/configuration.md | 8 +- docs/docutils.md | 2 +- docs/faq/index.md | 8 +- docs/intro.md | 4 +- docs/syntax/optional.md | 29 +-- docs/syntax/roles-and-directives.md | 6 +- docs/syntax/syntax.md | 219 ++++++++++++++++-- myst_parser/config/main.py | 29 +++ myst_parser/mdit_to_docutils/base.py | 151 +++++++++++- myst_parser/mdit_to_docutils/sphinx_.py | 20 ++ myst_parser/warnings_.py | 6 + tests/test_inventory.py | 16 ++ tests/test_renderers/fixtures/myst-config.txt | 73 ++++++ tests/test_renderers/test_myst_config.py | 6 +- 16 files changed, 523 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37a50b77..70bd6e94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -363,7 +363,7 @@ In particular for users, this update alters the parsing of tables to be consiste ### New Features ✨ -- **Task lists** utilise the [markdown-it-py tasklists plugin](markdown_it:md/plugins), and are applied to Markdown list items starting with `[ ]` or `[x]`. +- **Task lists** utilise the [markdown-it-py tasklists plugin](inv:markdown_it#md/plugins), and are applied to Markdown list items starting with `[ ]` or `[x]`. ```markdown - [ ] An item that needs doing @@ -541,7 +541,7 @@ substitutions: {{ key1 }} ``` -The substitutions are assessed as [jinja2 expressions](http://jinja.palletsprojects.com/) and includes the [Sphinx Environment](https://www.sphinx-doc.org/en/master/extdev/envapi.html) as `env`, so you can do powerful thinks like: +The substitutions are assessed as [jinja2 expressions](http://jinja.palletsprojects.com/) and includes the [Sphinx Environment](inv:sphinx#extdev/envapi) as `env`, so you can do powerful thinks like: ``` {{ [key1, env.docname] | join('/') }} diff --git a/docs/conf.py b/docs/conf.py index 301fa1f1..09726ff9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -97,6 +97,7 @@ "substitution", "tasklist", "attrs_inline", + "inv_link", ] myst_number_code_blocks = ["typescript"] myst_heading_anchors = 2 diff --git a/docs/configuration.md b/docs/configuration.md index 286a612a..8125d9a5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -69,6 +69,9 @@ Full details in the [](syntax/extensions) section. amsmath : enable direct parsing of [amsmath](https://ctan.org/pkg/amsmath) LaTeX equations +attrs_inline +: Enable inline attribute parsing, [see here](syntax/attributes) for details + colon_fence : Enable code fences using `:::` delimiters, [see here](syntax/colon_fence) for details @@ -87,6 +90,9 @@ html_admonition html_image : Convert HTML `<img>` elements to sphinx image nodes, [see here](syntax/images) for details +inv_link +: Enable the `inv:` schema for Markdown link destinations, [see here](syntax/inv_links) for details + linkify : Automatically identify "bare" web URLs and add hyperlinks @@ -117,7 +123,7 @@ WARNING: Non-consecutive header level increase; H1 to H3 [myst.header] **In general, if your build logs any warnings, you should either fix them or [raise an Issue](https://github.com/executablebooks/MyST-Parser/issues/new/choose) if you think the warning is erroneous.** -However, in some circumstances if you wish to suppress the warning you can use the [`suppress_warnings`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-suppress_warnings) configuration option, e.g. +However, in some circumstances if you wish to suppress the warning you can use the <inv:sphinx#suppress_warnings> configuration option, e.g. ```python suppress_warnings = ["myst.header"] diff --git a/docs/docutils.md b/docs/docutils.md index 10ea237e..9ec4aa15 100644 --- a/docs/docutils.md +++ b/docs/docutils.md @@ -36,7 +36,7 @@ The commands are based on the [Docutils Front-End Tools](https://docutils.source ::: :::{versionadded} 0.19.0 -`myst-suppress-warnings` replicates the functionality of sphinx's [`suppress_warnings`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-suppress_warnings) for `myst.` warnings in the `docutils` CLI. +`myst-suppress-warnings` replicates the functionality of sphinx's <inv:sphinx#suppress_warnings> for `myst.` warnings in the `docutils` CLI. ::: The CLI commands can also utilise the [`docutils.conf` configuration file](https://docutils.sourceforge.io/docs/user/config.html) to configure the behaviour of the CLI commands. For example: diff --git a/docs/faq/index.md b/docs/faq/index.md index 3f0c27c8..902dde6b 100644 --- a/docs/faq/index.md +++ b/docs/faq/index.md @@ -102,7 +102,7 @@ If you encounter any issues with this feature, please don't hesitate to report i (howto/autodoc)= ### Use `sphinx.ext.autodoc` in Markdown files -The [Sphinx extension `autodoc`](sphinx:sphinx.ext.autodoc), which pulls in code documentation from docstrings, is currently hard-coded to parse reStructuredText. +The [Sphinx extension `autodoc`](inv:sphinx#sphinx.ext.autodoc), which pulls in code documentation from docstrings, is currently hard-coded to parse reStructuredText. It is therefore incompatible with MyST's Markdown parser. However, the special [`eval-rst` directive](syntax/directives/parsing) can be used to "wrap" `autodoc` directives: @@ -142,7 +142,7 @@ See the [](syntax/header-anchors) section of extended syntaxes. ::: If you'd like to *automatically* generate targets for each of your section headers, -check out the {external+sphinx:std:doc}`autosectionlabel <usage/extensions/autosectionlabel>` +check out the [autosectionlabel](inv:sphinx#usage/*/autosectionlabel) sphinx feature. You can activate it in your Sphinx site by adding the following to your `conf.py` file: @@ -179,7 +179,7 @@ Moved to [](myst-warnings) ### Sphinx-specific page front matter Sphinx intercepts front matter and stores them within the global environment -(as discussed in the {external+sphinx:std:doc}`sphinx documentation <usage/restructuredtext/field-lists>`. +(as discussed in the [sphinx documentation](inv:sphinx#usage/*/field-lists)). There are certain front-matter keys (or their translations) that are also recognised specifically by docutils and parsed to inline Markdown: - `author` @@ -228,7 +228,7 @@ emphasis syntax will now be disabled. For example, the following will be rendere *emphasis is now disabled* ``` -For a list of all the syntax elements you can disable, see the [markdown-it parser guide](markdown_it:using). +For a list of all the syntax elements you can disable, see the [markdown-it parser guide](inv:markdown_it#using). ## Common errors and questions diff --git a/docs/intro.md b/docs/intro.md index f34725fb..671e4c5d 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -28,7 +28,7 @@ conda install -c conda-forge myst-parser (intro/sphinx)= ## Enable MyST in Sphinx -To get started with Sphinx, see their {external+sphinx:std:doc}`quick-start guide <usage/quickstart>`. +To get started with Sphinx, see their [quick-start guide](inv:sphinx#usage/quickstart). To use the MyST parser in Sphinx, simply add the following to your `conf.py` file: @@ -80,7 +80,7 @@ $ myst-docutils-html5 --stylesheet= myfile.md ``` To include this document within a Sphinx project, -include `myfile.md` in a [`toctree` directive](sphinx:toctree-directive) on an index page. +include `myfile.md` in a [`toctree` directive](inv:sphinx#toctree-directive) on an index page. ## Extend CommonMark with roles and directives diff --git a/docs/syntax/optional.md b/docs/syntax/optional.md index 4ad5073a..f9efca74 100644 --- a/docs/syntax/optional.md +++ b/docs/syntax/optional.md @@ -13,16 +13,16 @@ myst: :width: 200px ``` key4: example - confpy: sphinx `conf.py` {external+sphinx:std:doc}`configuration file <usage/configuration>` + confpy: sphinx `conf.py` [configuration file](inv:sphinx#usage/configuration) --- (syntax/extensions)= # Syntax Extensions -MyST-Parser is highly configurable, utilising the inherent "plugability" of the [markdown-it-py](markdown_it:index) parser. +MyST-Parser is highly configurable, utilising the inherent "plugability" of the [markdown-it-py](inv:markdown_it#index) parser. The following syntaxes are optional (disabled by default) and can be enabled *via* the sphinx `conf.py` (see also [](sphinx/config-options)). -Their goal is generally to add more *Markdown friendly* syntaxes; often enabling and rendering [markdown-it-py plugins](markdown_it:md/plugins) that extend the [CommonMark specification](https://commonmark.org/). +Their goal is generally to add more *Markdown friendly* syntaxes; often enabling and rendering [markdown-it-py plugins](inv:markdown_it#md/plugins) that extend the [CommonMark specification](https://commonmark.org/). To enable all the syntaxes explained below: @@ -36,6 +36,7 @@ myst_enable_extensions = [ "fieldlist", "html_admonition", "html_image", + "inv_link", "linkify", "replacements", "smartquotes", @@ -101,7 +102,7 @@ Math is parsed by adding to the `myst_enable_extensions` list option, in the {{ - `"dollarmath"` for parsing of dollar `$` and `$$` encapsulated math. - `"amsmath"` for direct parsing of [amsmath LaTeX environments](https://ctan.org/pkg/amsmath). -These options enable their respective Markdown parser plugins, as detailed in the [markdown-it plugin guide](markdown_it:md/plugins). +These options enable their respective Markdown parser plugins, as detailed in the [markdown-it plugin guide](inv:markdown_it#md/plugins). :::{versionchanged} 0.13.0 `myst_dmath_enable=True` and `myst_amsmath_enable=True` are deprecated, and replaced by `myst_enable_extensions = ["dollarmath", "amsmath"]` @@ -231,7 +232,7 @@ See [the extended syntax option](syntax/amsmath). (syntax/mathjax)= ### Mathjax and math parsing -When building HTML using the {external+sphinx:mod}`sphinx.ext.mathjax <sphinx.ext.mathjax>` extension (enabled by default), +When building HTML using the <inv:sphinx#sphinx.ext.mathjax> extension (enabled by default), If `dollarmath` is enabled, Myst-Parser injects the `tex2jax_ignore` (MathJax v2) and `mathjax_ignore` (MathJax v3) classes in to the top-level section of each MyST document, and adds the following default MathJax configuration: MathJax version 2 (see [the tex2jax preprocessor](https://docs.mathjax.org/en/v2.7-latest/options/preprocessors/tex2jax.html#configure-tex2jax): @@ -353,7 +354,7 @@ This may lead to unexpected outcomes. ::: -Substitution references are assessed as [Jinja2 expressions](http://jinja.palletsprojects.com) which can use [filters](https://jinja.palletsprojects.com/en/2.11.x/templates/#list-of-builtin-filters), and also contains the {external+sphinx:std:doc}`Sphinx Environment <extdev/envapi>` in the context (as `env`). +Substitution references are assessed as [Jinja2 expressions](http://jinja.palletsprojects.com) which can use [filters](https://jinja.palletsprojects.com/en/2.11.x/templates/#list-of-builtin-filters), and also contains the [Sphinx Environment](inv:sphinx#extdev/envapi) in the context (as `env`). Therefore you can do things like: ```md @@ -540,7 +541,7 @@ $ myst-anchors -l 2 docs/syntax/optional.md By adding `"deflist"` to `myst_enable_extensions` (in the {{ confpy }}), you will be able to utilise definition lists. -Definition lists utilise the [markdown-it-py deflist plugin](markdown_it:md/plugins), which itself is based on the [Pandoc definition list specification](http://johnmacfarlane.net/pandoc/README.html#definition-lists). +Definition lists utilise the [markdown-it-py deflist plugin](inv:markdown_it#md/plugins), which itself is based on the [Pandoc definition list specification](http://johnmacfarlane.net/pandoc/README.html#definition-lists). This syntax can be useful, for example, as an alternative to nested bullet-lists: @@ -619,7 +620,7 @@ Term 3 By adding `"tasklist"` to `myst_enable_extensions` (in the {{ confpy }}), you will be able to utilise task lists. -Task lists utilise the [markdown-it-py tasklists plugin](markdown_it:md/plugins), +Task lists utilise the [markdown-it-py tasklists plugin](inv:markdown_it#md/plugins), and are applied to markdown list items starting with `[ ]` or `[x]`: ```markdown @@ -691,7 +692,7 @@ based on the [reStructureText syntax](https://docutils.sourceforge.io/docs/ref/r print("Hello, world!") ``` -A prominent use case of field lists is for use in API docstrings, as used in [Sphinx's docstring renderers](sphinx:python-domain): +A prominent use case of field lists is for use in API docstrings, as used in [Sphinx's docstring renderers](inv:sphinx#python-domain): ````md ```{py:function} send_message(sender, priority) @@ -726,16 +727,16 @@ Currently `sphinx.ext.autodoc` does not support MyST, see [](howto/autodoc). (syntax/attributes)= ## Inline attributes +:::{versionadded} 0.19 +This feature is in *beta*, and may change in future versions. +It replace the previous `attrs_image` extension, which is now deprecated. +::: + By adding `"attrs_inline"` to `myst_enable_extensions` (in the {{ confpy }}), you can enable parsing of inline attributes after certain inline syntaxes. This is adapted from [djot inline attributes](https://htmlpreview.github.io/?https://github.com/jgm/djot/blob/master/doc/syntax.html#inline-attributes), and also related to [pandoc bracketed spans](https://pandoc.org/MANUAL.html#extension-bracketed_spans). -:::{important} -This feature is in *beta*, and may change in future versions. -It replace the previous `attrs_image` extension, which is now deprecated. -::: - Attributes are specified in curly braces after the inline syntax. Inside the curly braces, the following syntax is recognised: diff --git a/docs/syntax/roles-and-directives.md b/docs/syntax/roles-and-directives.md index 3017b527..df760679 100644 --- a/docs/syntax/roles-and-directives.md +++ b/docs/syntax/roles-and-directives.md @@ -8,8 +8,8 @@ by interpreting a chuck of text as a specific type of markup, according to its n Mostly all [docutils roles](https://docutils.sourceforge.io/docs/ref/rst/roles.html), [docutils directives](https://docutils.sourceforge.io/docs/ref/rst/directives.html), -{external+sphinx:std:doc}`Sphinx roles <usage/restructuredtext/roles>`, or -{external+sphinx:std:doc}`Sphinx directives <usage/restructuredtext/directives>` +[Sphinx roles](inv:sphinx#usage/*/roles), or +[Sphinx directives](inv:sphinx#usage/*/directives) can be used in MyST. ## Syntax @@ -421,6 +421,6 @@ For example: > {sub-ref}`today` | {sub-ref}`wordcount-words` words | {sub-ref}`wordcount-minutes` min read -`today` is replaced by either the date on which the document is parsed, with the format set by [`today_fmt`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-today_fmt), or the `today` variable if set in the configuration file. +`today` is replaced by either the date on which the document is parsed, with the format set by <inv:sphinx#today_fmt>, or the `today` variable if set in the configuration file. The reading speed is computed using the `myst_words_per_minute` configuration (see the [Sphinx configuration options](sphinx/config-options)). diff --git a/docs/syntax/syntax.md b/docs/syntax/syntax.md index 0062a551..5d3ac61d 100644 --- a/docs/syntax/syntax.md +++ b/docs/syntax/syntax.md @@ -85,7 +85,7 @@ would be equivalent to: ### Setting HTML Metadata The front-matter can contain the special key `html_meta`; a dict with data to add to the generated HTML as [`<meta>` elements](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta). -This is equivalent to using the {external+sphinx:ref}`meta directive <html-meta>`. +This is equivalent to using the [meta directive](inv:sphinx#html-meta). HTML metadata can also be added globally in the `conf.py` *via* the `myst_html_meta` variable, in which case it will be added to all MyST documents. For each document, the `myst_html_meta` dict will be updated by the document level front-matter `html_meta`, with the front-matter taking precedence. @@ -207,34 +207,203 @@ Is below, but it won't be parsed into the document. ## Markdown Links and Referencing -Markdown links are of the form: `[text](link)`. +### CommonMark link format -If you set the configuration `myst_all_links_external = True` (`False` by default), -then all links will be treated simply as "external" links. -For example, in HTML outputs, `[text](link)` will be rendered as `<a href="link">text</a>`. +CommonMark links come in three forms ([see the spec](https://spec.commonmark.org/0.30/#links)): -Otherwise, links will only be treated as "external" links if they are prefixed with a scheme, -configured with `myst_url_schemes` (by default, `http`, `https`, `ftp`, or `mailto`). -For example, `[example.com](https://example.com)` becomes [example.com](https://example.com). +*Autolinks* are [URIs][uri] surrounded by `<` and `>`, which must always have a scheme: -:::{note} -The `text` will be parsed as nested Markdown, for example `[here's some *emphasised text*](https://example.com)` will be parsed as [here's some *emphasised text*](https://example.com). +```md +<scheme:path?query#fragment> +``` + +*Inline links* allow for optional explicit text and titles (in HTML titles are rendered as tooltips): + +```md +[Explicit *Markdown* text](destination "optional explicit title") +``` + +or, if the destination contains spaces, + +```md +[Explicit *Markdown* text](<a destination> "optional explicit title") +``` + +*Reference links* define the destination separately in the document, and can be used multiple times: + +```md +[Explicit *Markdown* text][label] +[Another link][label] + +[label]: destination "optional explicit title" +``` + +[uri]: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier +[url]: https://en.wikipedia.org/wiki/URL + +### Default destination resolution + +The destination of a link can resolve to either an **external** target, such as a [URL] to another website, +or an **internal** target, such as a file, heading or figure within the same project. + +By default, MyST will resolve link destinations according to the following rules: + +1. All autolinks will be treated as external [URL] links. + +2. Destinations beginning with `http:`, `https:`, `ftp:`, or `mailto:` will be treated as external [URL] links. + +3. Destinations which point to a local file path are treated as links to that file. + - The path must be relative and in [POSIX format](https://en.wikipedia.org/wiki/Path_(computing)#POSIX_and_Unix_paths) (i.e. `/` separators). + - If the path is to another source file in the project (e.g. a `.md` or `.rst` file), + then the link will be to the initial heading in that file. + - If the path is to a non-source file (e.g. a `.png` or `.pdf` file), + then the link will be to the file itself, e.g. to download it. + +4. Destinations beginning with `#` will be treated as a link to a heading "slug" in the same file. + - This requires the `myst_heading_anchors` configuration be set. + - For more details see [](syntax/header-anchors). + +5. All other destinations are treated as internal references, which can link to any type of target within the project (see [](syntax/targets)). + +Here are some examples: + +:::{list-table} +:header-rows: 1 + +* - Type + - Syntax + - Rendered + +* - Autolink + - `<https://example.com>` + - <https://example.com> + +* - External URL + - `[example.com](https://example.com)` + - [example.com](https://example.com) + +* - Internal source file + - `[Source file](syntax.md)` + - [Source file](syntax.md) + +* - Internal non-source file + - `[Non-source file](example.txt)` + - [Non-source file](example.txt) + +* - Internal heading + - `[Heading](#markdown-links-and-referencing)` + - [Heading](#markdown-links-and-referencing) + +::: + +### Customising destination resolution + +You can customise the default destination resolution rules by setting the following [configuration options](../configuration.md): + +`myst_all_links_external` (default: `False`) +: If `True`, then all links will be treated as external links. + +`myst_url_schemes` (default: `["http", "https", "ftp", "mailto"]`) +: A list of [URL] schemes which will be treated as external links. + +`myst_ref_domains` (default: `[]`) +: A list of [sphinx domains](inv:sphinx#domain) which will be allowed for internal links. + For example, `myst_ref_domains = ("std", "py")` will only allow cross-references to `std` and `py` domains. + If the list is empty, then all domains will be allowed. + +(syntax/inv_links)= +### Cross-project (inventory) links + +:::{versionadded} 0.19 +This functionality is currently in *beta*. +It is intended that eventually it will be part of the core syntax. ::: -For "internal" links, myst-parser in Sphinx will attempt to resolve the reference to either a relative document path, or a cross-reference to a target (see [](syntax/targets)): +Each Sphinx HTML build creates a file named `objects.inv` that contains a mapping from referenceable objects to [URIs][uri] relative to the HTML set’s root. +Each object is uniquely identified by a `domain`, `type`, and `name`. +As well as the relative location, the object can also include implicit `text` for the reference (like the text for a heading). + +You can use the `myst-inv` command line tool (installed with `myst_parser`) to visualise and filter any remote URL or local file path to this inventory file (or its parent): + +```yaml +# $ myst-inv https://www.sphinx-doc.org/en/master -n index +name: Sphinx +version: 6.2.0 +base_url: https://www.sphinx-doc.org/en/master +objects: + rst: + role: + index: + loc: usage/restructuredtext/directives.html#role-index + text: null + std: + doc: + index: + loc: index.html + text: Welcome +``` + +To load external inventories into your Sphinx project, you must load the [`sphinx.ext.intersphinx` extension](inv:sphinx#usage/*/intersphinx), and set the `intersphinx_mapping` configuration option. +Then also enable the `inv_link` MyST extension e.g.: + +```python +extensions = ["myst_parser", "sphinx.ext.intersphinx"] +intersphinx_mapping = { + "sphinx": ("https://www.sphinx-doc.org/en/master", None), +} +myst_enable_extensions = ["inv_link"] +``` + +:::{dropdown} Docutils configuration -- `[this doc](syntax.md)` will link to a rendered source document: [this doc](syntax.md) - - This is similar to `` {doc}`this doc <syntax>` ``; {doc}`this doc <syntax>`, but allows for document extensions, and parses nested Markdown text. -- `[example text](example.txt)` will link to a non-source (downloadable) file: [example text](example.txt) - - The linked document itself will be copied to the build directory. - - This is similar to `` {download}`example text <example.txt>` ``; {download}`example text <example.txt>`, but parses nested Markdown text. -- `[reference](syntax/referencing)` will link to an internal cross-reference: [reference](syntax/referencing) - - This is similar to `` {any}`reference <syntax/referencing>` ``; {any}`reference <syntax/referencing>`, but parses nested Markdown text. - - You can limit the scope of the cross-reference to specific [sphinx domains](sphinx:domain), by using the `myst_ref_domains` configuration. - For example, `myst_ref_domains = ("std", "py")` will only allow cross-references to `std` and `py` domains. +Use the `docutils.conf` configuration file, for more details see [](myst-docutils). -Additionally, only if [](syntax/header-anchors) are enabled, then internal links to document headers can be used. -For example `[a header](syntax.md#markdown-links-and-referencing)` will link to a header anchor: [a header](syntax.md#markdown-links-and-referencing). +```ini +[general] +myst-inventories: + sphinx: ["https://www.sphinx-doc.org/en/master", null] +myst-enable-extensions: inv_link +``` + +::: + +you can then reference inventory objects by prefixing the `inv` schema to the destination [URI]: `inv:key:domain:type#name`. + +`key`, `domain` and `type` are optional, e.g. for `inv:#name`, all inventories, domains and types will be searched, with a [warning emitted](myst-warnings) if multiple matches are found. + +Additionally, `*` is a wildcard which matches zero or characters, e.g. `inv:*:std:doc#a*` will match all `std:doc` objects in all inventories, with a `name` beginning with `a`. +Note, to match to a literal `*` use `\*`. + +Here are some examples: + +:::{list-table} +:header-rows: 1 + +* - Type + - Syntax + - Rendered + +* - Autolink, full + - `<inv:sphinx:std:doc#index>` + - <inv:sphinx:std:doc#index> + +* - Link, full + - `[Sphinx](inv:sphinx:std:doc#index)` + - [Sphinx](inv:sphinx:std:doc#index) + +* - Autolink, no type + - `<inv:sphinx:std#index>` + - <inv:sphinx:std#index> + +* - Autolink, no domain + - `<inv:sphinx:*:doc#index>` + - <inv:sphinx:*:doc#index> + +* - Autolink, only name + - `<inv:#*.Sphinx>` + - <inv:#*.Sphinx> + +::: (syntax/targets)= @@ -258,7 +427,7 @@ Target headers are defined with this syntax: ``` They can then be referred to with the -{external+sphinx:ref}`ref inline role <ref-role>`: +[`ref` inline role](inv:sphinx#ref-role): ```md {ref}`header_target` @@ -278,7 +447,7 @@ Alternatively using the markdown syntax: [my text](header_target) ``` -is equivalent to using the {external+sphinx:ref}`any inline role <any-role>`: +is equivalent to using the [`any` inline role](inv:sphinx#any-role): ```md {any}`my text <header_target>` @@ -314,7 +483,7 @@ c = "string" ``` You can create and register your own lexer, using the [`pygments.lexers` entry point](https://pygments.org/docs/plugins/#register-plugins), -or within a sphinx extension, with the [`app.add_lexer` method](sphinx:sphinx.application.Sphinx.add_lexer). +or within a sphinx extension, with the [`app.add_lexer` method](inv:sphinx#*.Sphinx.add_lexer). Using the `myst_number_code_blocks` configuration option, you can also control whether code blocks are numbered by line. For example, using `myst_number_code_blocks = ["typescript"]`: diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py index 5ca5ec44..7b36b7fc 100644 --- a/myst_parser/config/main.py +++ b/myst_parser/config/main.py @@ -41,6 +41,7 @@ def check_extensions(_, field: dc.Field, value: Any): "fieldlist", "html_admonition", "html_image", + "inv_link", "linkify", "replacements", "smartquotes", @@ -64,6 +65,23 @@ def check_sub_delimiters(_, field: dc.Field, value: Any): ) +def check_inventories(_, field: dc.Field, value: Any): + """Check that the inventories are a dict of {str: (str, Optional[str])}""" + if not isinstance(value, dict): + raise TypeError(f"'{field.name}' is not a dictionary: {value!r}") + for key, val in value.items(): + if not isinstance(key, str): + raise TypeError(f"'{field.name}' key is not a string: {key!r}") + if not isinstance(val, (tuple, list)) or len(val) != 2: + raise TypeError( + f"'{field.name}[{key}]' value is not a 2-item list: {val!r}" + ) + if not isinstance(val[0], str): + raise TypeError(f"'{field.name}[{key}][0]' is not a string: {val[0]}") + if not (val[1] is None or isinstance(val[1], str)): + raise TypeError(f"'{field.name}[{key}][1]' is not a null/string: {val[1]}") + + @dc.dataclass() class MdParserConfig: """Configuration options for the Markdown Parser. @@ -304,6 +322,17 @@ class MdParserConfig: }, ) + inventories: Dict[str, Tuple[str, Optional[str]]] = dc.field( + default_factory=dict, + repr=False, + metadata={ + "validator": check_inventories, + "help": "Mapping of key to (url, inv file), for intra-project referencing", + "docutils_only": True, + "global_only": True, + }, + ) + def __post_init__(self): validate_fields(self) diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 2e8096fc..b72ef5f3 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -4,9 +4,10 @@ import inspect import json import os +import posixpath import re from collections import OrderedDict -from contextlib import contextmanager +from contextlib import contextmanager, suppress from datetime import date, datetime from types import ModuleType from typing import ( @@ -40,6 +41,7 @@ from markdown_it.token import Token from markdown_it.tree import SyntaxTreeNode +from myst_parser import inventory from myst_parser._compat import findall from myst_parser.config.main import MdParserConfig from myst_parser.mocking import ( @@ -93,6 +95,8 @@ def __init__(self, parser: MarkdownIt) -> None: for k, v in inspect.getmembers(self, predicate=inspect.ismethod) if k.startswith("render_") and k != "render_children" } + # these are lazy loaded, when needed + self._inventories: None | dict[str, inventory.InventoryType] = None def __getattr__(self, name: str): """Warn when the renderer has not been setup yet.""" @@ -727,18 +731,27 @@ def render_link(self, token: SyntaxTreeNode) -> None: or any scheme if `myst_url_schemes` is None. - Otherwise, forward to `render_internal_link` """ - if token.info == "auto": # handles both autolink and linkify - return self.render_autolink(token) - if ( self.md_config.commonmark_only or self.md_config.gfm_only or self.md_config.all_links_external ): - return self.render_external_url(token) + if token.info == "auto": # handles both autolink and linkify + return self.render_autolink(token) + else: + return self.render_external_url(token) + + href = cast(str, token.attrGet("href") or "") + + # TODO ideally whether inv_link is enabled could be precomputed + if "inv_link" in self.md_config.enable_extensions and href.startswith("inv:"): + return self.create_inventory_link(token) + + if token.info == "auto": # handles both autolink and linkify + return self.render_autolink(token) # Check for external URL - url_scheme = urlparse(cast(str, token.attrGet("href") or "")).scheme + url_scheme = urlparse(href).scheme allowed_url_schemes = self.md_config.url_schemes if (allowed_url_schemes is None and url_scheme) or ( allowed_url_schemes is not None and url_scheme in allowed_url_schemes @@ -797,6 +810,132 @@ def render_autolink(self, token: SyntaxTreeNode) -> None: with self.current_node_context(ref_node, append=True): self.render_children(token) + def create_inventory_link(self, token: SyntaxTreeNode) -> None: + r"""Create a link to an inventory object. + + This assumes the href is of the form `<scheme>:<path>#<target>`. + The path is of the form `<invs>:<domains>:<otypes>`, + where each of the parts is optional, hence `<scheme>:#<target>` is also valid. + Each of the path parts can contain the `*` wildcard, for example: + `<scheme>:key:*:obj#targe*`. + `\*` is treated as a plain `*`. + """ + + # account for autolinks + if token.info == "auto": + # autolinks escape the HTML, which we don't want + href = token.children[0].content + explicit = False + else: + href = cast(str, token.attrGet("href") or "") + explicit = bool(token.children) + + # split the href up into parts + uri_parts = urlparse(href) + target = uri_parts.fragment + invs, domains, otypes = None, None, None + if uri_parts.path: + path_parts = uri_parts.path.split(":") + with suppress(IndexError): + invs = path_parts[0] + domains = path_parts[1] + otypes = path_parts[2] + + # find the matches + matches = self.get_inventory_matches( + target=target, invs=invs, domains=domains, otypes=otypes + ) + + # warn for 0 or >1 matches + if not matches: + filter_str = inventory.filter_string(invs, domains, otypes, target) + self.create_warning( + f"No matches for {filter_str!r}", + MystWarnings.IREF_MISSING, + line=token_line(token, default=0), + append_to=self.current_node, + ) + return + if len(matches) > 1: + show_num = 3 + filter_str = inventory.filter_string(invs, domains, otypes, target) + matches_str = ", ".join( + [ + inventory.filter_string(m.inv, m.domain, m.otype, m.name) + for m in matches[:show_num] + ] + ) + if len(matches) > show_num: + matches_str += ", ..." + self.create_warning( + f"Multiple matches for {filter_str!r}: {matches_str}", + MystWarnings.IREF_AMBIGUOUS, + line=token_line(token, default=0), + append_to=self.current_node, + ) + + # create the docutils node + match = matches[0] + ref_node = nodes.reference() + ref_node["internal"] = False + ref_node["inv_match"] = inventory.filter_string( + match.inv, match.domain, match.otype, match.name + ) + self.add_line_and_source_path(ref_node, token) + self.copy_attributes( + token, ref_node, ("class", "id", "reftitle"), aliases={"title": "reftitle"} + ) + ref_node["refuri"] = ( + posixpath.join(match.base_url, match.loc) if match.base_url else match.loc + ) + if "reftitle" not in ref_node: + ref_node["reftitle"] = f"{match.project} {match.version}".strip() + self.current_node.append(ref_node) + if explicit: + with self.current_node_context(ref_node): + self.render_children(token) + elif match.text: + ref_node.append(nodes.Text(match.text)) + else: + ref_node.append(nodes.Text(match.name)) + + def get_inventory_matches( + self, + *, + invs: str | None, + domains: str | None, + otypes: str | None, + target: str | None, + ) -> list[inventory.InvMatch]: + """Return inventory matches. + + This will be overridden for sphinx, to use intersphinx config. + """ + if self._inventories is None: + self._inventories = {} + for key, (uri, path) in self.md_config.inventories.items(): + load_path = posixpath.join(uri, "objects.inv") if path is None else path + self.reporter.info(f"Loading inventory {key!r}: {load_path}") + try: + inv = inventory.fetch_inventory(load_path, base_url=uri) + except Exception as exc: + self.create_warning( + f"Failed to load inventory {key!r}: {exc}", + MystWarnings.INV_LOAD, + ) + else: + self._inventories[key] = inv + + return list( + inventory.filter_inventories( + self._inventories, + invs=invs, + domains=domains, + otypes=otypes, + targets=target, + ) + ) + def render_html_inline(self, token: SyntaxTreeNode) -> None: self.render_html_block(token) diff --git a/myst_parser/mdit_to_docutils/sphinx_.py b/myst_parser/mdit_to_docutils/sphinx_.py index 3fed1224..c1989edb 100644 --- a/myst_parser/mdit_to_docutils/sphinx_.py +++ b/myst_parser/mdit_to_docutils/sphinx_.py @@ -13,9 +13,11 @@ from sphinx.domains.math import MathDomain from sphinx.domains.std import StandardDomain from sphinx.environment import BuildEnvironment +from sphinx.ext.intersphinx import InventoryAdapter from sphinx.util import logging from sphinx.util.nodes import clean_astext +from myst_parser import inventory from myst_parser.mdit_to_docutils.base import DocutilsRenderer from myst_parser.warnings_ import MystWarnings @@ -92,6 +94,24 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: with self.current_node_context(inner_node): self.render_children(token) + def get_inventory_matches( + self, + *, + invs: str | None, + domains: str | None, + otypes: str | None, + target: str | None, + ) -> list[inventory.InvMatch]: + return list( + inventory.filter_sphinx_inventories( + InventoryAdapter(self.sphinx_env).named_inventory, + invs=invs, + domains=domains, + otypes=otypes, + targets=target, + ) + ) + def render_heading(self, token: SyntaxTreeNode) -> None: """This extends the docutils method, to allow for the addition of heading ids. These ids are computed by the ``markdown-it-py`` ``anchors_plugin`` diff --git a/myst_parser/warnings_.py b/myst_parser/warnings_.py index 78b94914..6325c4af 100644 --- a/myst_parser/warnings_.py +++ b/myst_parser/warnings_.py @@ -32,6 +32,12 @@ class MystWarnings(Enum): # cross-reference resolution XREF_AMBIGUOUS = "xref_ambiguous" """Multiple targets were found for a cross-reference.""" + INV_LOAD = "inv_retrieval" + """Failure to retrieve or load an inventory.""" + IREF_MISSING = "iref_missing" + """A target was not found for an inventory reference.""" + IREF_AMBIGUOUS = "iref_ambiguous" + """Multiple targets were found for an inventory reference.""" LEGACY_DOMAIN = "domains" """A legacy domain found, which does not support `resolve_any_xref`.""" diff --git a/tests/test_inventory.py b/tests/test_inventory.py index 795a1144..825bcccb 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -3,6 +3,7 @@ import pytest +from myst_parser.config.main import MdParserConfig from myst_parser.inventory import ( filter_inventories, from_sphinx, @@ -14,6 +15,21 @@ STATIC = Path(__file__).parent.absolute() / "static" +@pytest.mark.parametrize( + "value", + [ + None, + {1: 2}, + {"key": 1}, + {"key": [1, 2]}, + {"key": ["a", 1]}, + ], +) +def test_docutils_config_invalid(value): + with pytest.raises((TypeError, ValueError)): + MdParserConfig(inventories=value) + + def test_convert_roundtrip(): with (STATIC / "objects_v2.inv").open("rb") as f: inv = load(f) diff --git a/tests/test_renderers/fixtures/myst-config.txt b/tests/test_renderers/fixtures/myst-config.txt index 562f99b1..0f898cb5 100644 --- a/tests/test_renderers/fixtures/myst-config.txt +++ b/tests/test_renderers/fixtures/myst-config.txt @@ -256,3 +256,76 @@ text <string>:1: (WARNING/2) Invalid 'height' attribute value: '2x' [myst.attribute] <string>:1: (WARNING/2) Invalid 'align' attribute value: 'other' [myst.attribute] . + +[inv_link] --myst-enable-extensions=inv_link +. +<inv:#index> +[](inv:#index) +[*explicit*](inv:#index) +<inv:key#index> +[](inv:key#index) +<inv:key:std:label#search> +[](inv:key:std:label#search) +<inv:#in*> +[](inv:#in*) +<inv:key:*:doc#index> +[](inv:key:*:doc#index) +. +<document source="<string>"> + <paragraph> + <reference internal="False" inv_match="key:std:doc:index" reftitle="Python" refuri="https://example.com/index.html"> + Title + + <reference internal="False" inv_match="key:std:doc:index" reftitle="Python" refuri="https://example.com/index.html"> + Title + + <reference internal="False" inv_match="key:std:doc:index" reftitle="Python" refuri="https://example.com/index.html"> + <emphasis> + explicit + + <reference internal="False" inv_match="key:std:doc:index" reftitle="Python" refuri="https://example.com/index.html"> + Title + + <reference internal="False" inv_match="key:std:doc:index" reftitle="Python" refuri="https://example.com/index.html"> + Title + + <reference internal="False" inv_match="key:std:label:search" reftitle="Python" refuri="https://example.com/search.html"> + Search Page + + <reference internal="False" inv_match="key:std:label:search" reftitle="Python" refuri="https://example.com/search.html"> + Search Page + + <reference internal="False" inv_match="key:std:doc:index" reftitle="Python" refuri="https://example.com/index.html"> + Title + + <reference internal="False" inv_match="key:std:doc:index" reftitle="Python" refuri="https://example.com/index.html"> + Title + + <reference internal="False" inv_match="key:std:doc:index" reftitle="Python" refuri="https://example.com/index.html"> + Title + + <reference internal="False" inv_match="key:std:doc:index" reftitle="Python" refuri="https://example.com/index.html"> + Title +. + +[inv_link_error] --myst-enable-extensions=inv_link +. +<inv:#other> + +<inv:*:*:*#*index> +. +<document source="<string>"> + <paragraph> + <system_message level="2" line="1" source="<string>" type="WARNING"> + <paragraph> + No matches for '*:*:*:other' [myst.iref_missing] + <paragraph> + <system_message level="2" line="3" source="<string>" type="WARNING"> + <paragraph> + Multiple matches for '*:*:*:*index': key:std:label:genindex, key:std:label:modindex, key:std:label:py-modindex, ... [myst.iref_ambiguous] + <reference internal="False" inv_match="key:std:label:genindex" reftitle="Python" refuri="https://example.com/genindex.html"> + Index + +<string>:1: (WARNING/2) No matches for '*:*:*:other' [myst.iref_missing] +<string>:3: (WARNING/2) Multiple matches for '*:*:*:*index': key:std:label:genindex, key:std:label:modindex, key:std:label:py-modindex, ... [myst.iref_ambiguous] +. diff --git a/tests/test_renderers/test_myst_config.py b/tests/test_renderers/test_myst_config.py index 0640238a..ae8d9519 100644 --- a/tests/test_renderers/test_myst_config.py +++ b/tests/test_renderers/test_myst_config.py @@ -5,14 +5,16 @@ import pytest from docutils.core import Publisher, publish_string +from pytest_param_files import ParamTestData from myst_parser.parsers.docutils_ import Parser FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures") +INV_PATH = Path(__file__).parent.parent.absolute() / "static" / "objects_v2.inv" @pytest.mark.param_file(FIXTURE_PATH / "myst-config.txt") -def test_cmdline(file_params): +def test_cmdline(file_params: ParamTestData): """The description is parsed as a docutils commandline""" pub = Publisher(parser=Parser()) option_parser = pub.setup_option_parser() @@ -27,6 +29,8 @@ def test_cmdline(file_params): report_stream = StringIO() settings["output_encoding"] = "unicode" settings["warning_stream"] = report_stream + if "inv_" in file_params.title: + settings["myst_inventories"] = {"key": ["https://example.com", str(INV_PATH)]} output = publish_string( file_params.content, parser=Parser(), From 797af5f66cbaf52bf98f9f2a398163d24b1ac0d2 Mon Sep 17 00:00:00 2001 From: Matthieu MOREL <matthieu.morel35@gmail.com> Date: Tue, 10 Jan 2023 06:55:04 +0100 Subject: [PATCH 15/20] =?UTF-8?q?=F0=9F=94=A7=20ci(deps):=20setup=20depend?= =?UTF-8?q?abot=20(#669)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Chris Sewell <chrisj_sewell@hotmail.com> --- .github/dependabot.yml | 19 +++++++++++++++++++ .github/workflows/test-formats.yml | 8 ++++---- .github/workflows/tests.yml | 24 ++++++++++++------------ 3 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..786be571 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + commit-message: + prefix: ⬆️ + schedule: + interval: weekly + - package-ecosystem: pip + directory: / + commit-message: + prefix: ⬆️ + schedule: + interval: weekly diff --git a/.github/workflows/test-formats.yml b/.github/workflows/test-formats.yml index 3bb31c6e..6b0bf594 100644 --- a/.github/workflows/test-formats.yml +++ b/.github/workflows/test-formats.yml @@ -18,9 +18,9 @@ jobs: format: ["man", "text"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 - name: Install dependencies @@ -42,9 +42,9 @@ jobs: format: ["latex"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 - name: Install dependencies diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 07aac7b1..2429041f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.8 - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.8" - - uses: pre-commit/action@v2.0.0 + - uses: pre-commit/action@v3.0.0 tests: @@ -39,9 +39,9 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -54,7 +54,7 @@ jobs: coverage xml - name: Upload to Codecov if: github.repository == 'executablebooks/MyST-Parser' && matrix.python-version == 3.8 && matrix.os == 'ubuntu-latest' - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: name: myst-parser-pytests flags: pytests @@ -73,9 +73,9 @@ jobs: steps: - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.8 - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.8" - name: Install setup @@ -109,9 +109,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.8 - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.8" - name: install flit @@ -132,9 +132,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.8 - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.8" - name: install flit and tomlkit From 8daa00b89be0fe51dd028985eeb762bb8fe13642 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Wed, 11 Jan 2023 11:47:48 +0100 Subject: [PATCH 16/20] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Allow=20for=20h?= =?UTF-8?q?eading=20anchor=20links=20in=20docutils=20(#678)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This aligns the treatment of `[](#target)` style links for docutils with sphinx, such that they are linked to a heading slug. The core behaviour for sphinx is not changed, except that failed reference resolution now emits a `myst.xref_missing` warning (as opposed to a `std.ref` one), with a clearer warning message. Also on failure, the reference is still created, for people who wish to suppress the warning (see e.g. #677) --- docs/intro.md | 2 +- docs/syntax/syntax.md | 9 +- myst_parser/config/main.py | 3 +- myst_parser/mdit_to_docutils/base.py | 109 ++++++++++---- myst_parser/mdit_to_docutils/sphinx_.py | 99 ++++--------- myst_parser/sphinx_ext/myst_refs.py | 140 ++++++++++-------- myst_parser/warnings_.py | 2 + .../fixtures/docutil_syntax_elements.md | 5 +- tests/test_renderers/fixtures/myst-config.txt | 14 ++ .../fixtures/sphinx_syntax_elements.md | 10 +- .../test_myst_refs/doc_with_extension.xml | 2 +- .../test_heading_slug_func.resolved.xml | 4 +- .../test_heading_slug_func.xml | 4 +- .../test_sphinx_builds/test_includes.html | 2 +- .../test_sphinx_builds/test_includes.xml | 2 +- .../test_sphinx_builds/test_references.html | 16 +- .../test_references.resolved.xml | 22 +-- .../test_sphinx_builds/test_references.xml | 23 +-- .../test_references_singlehtml.html | 10 +- .../test_references_singlehtml.resolved.xml | 6 +- .../test_references_singlehtml.xml | 6 +- 21 files changed, 274 insertions(+), 216 deletions(-) diff --git a/docs/intro.md b/docs/intro.md index 671e4c5d..c4884960 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -46,7 +46,7 @@ To parse single documents, see the [](docutils.md) section ## Write a CommonMark document MyST is an extension of [CommonMark Markdown](https://commonmark.org/), -that includes [additional syntax](../syntax/syntax.md) for technical authoring, +that includes [additional syntax](syntax/syntax.md) for technical authoring, which integrates with Docutils and Sphinx. To start off, create an empty file called `myfile.md` and give it a markdown title and text. diff --git a/docs/syntax/syntax.md b/docs/syntax/syntax.md index 5d3ac61d..c2f5b277 100644 --- a/docs/syntax/syntax.md +++ b/docs/syntax/syntax.md @@ -255,7 +255,8 @@ By default, MyST will resolve link destinations according to the following rules 3. Destinations which point to a local file path are treated as links to that file. - The path must be relative and in [POSIX format](https://en.wikipedia.org/wiki/Path_(computing)#POSIX_and_Unix_paths) (i.e. `/` separators). - If the path is to another source file in the project (e.g. a `.md` or `.rst` file), - then the link will be to the initial heading in that file. + then the link will be to the initial heading in that file or, + if the path is appended by a `#target`, to the heading "slug" in that file. - If the path is to a non-source file (e.g. a `.png` or `.pdf` file), then the link will be to the file itself, e.g. to download it. @@ -290,10 +291,14 @@ Here are some examples: - `[Non-source file](example.txt)` - [Non-source file](example.txt) -* - Internal heading +* - Local heading - `[Heading](#markdown-links-and-referencing)` - [Heading](#markdown-links-and-referencing) +* - Heading in another file + - `[Heading](optional.md#auto-generated-header-anchors)` + - [Heading](optional.md#auto-generated-header-anchors) + ::: ### Customising destination resolution diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py index 7b36b7fc..7acc3c55 100644 --- a/myst_parser/config/main.py +++ b/myst_parser/config/main.py @@ -170,7 +170,6 @@ class MdParserConfig: metadata={ "validator": optional(in_([1, 2, 3, 4, 5, 6, 7])), "help": "Heading level depth to assign HTML anchors", - "sphinx_only": True, }, ) @@ -180,7 +179,7 @@ class MdParserConfig: "validator": optional(is_callable), "help": "Function for creating heading anchors", "global_only": True, - "sphinx_only": True, + "sphinx_only": True, # TODO docutils config doesn't handle callables }, ) diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index b72ef5f3..1c6a0010 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -136,6 +136,8 @@ def setup_render( self._level_to_elem: dict[int, nodes.document | nodes.section] = { 0: self.document } + # mapping of section slug to section node + self._slug_to_section: dict[str, nodes.section] = {} @property def sphinx_env(self) -> BuildEnvironment | None: @@ -236,6 +238,37 @@ def _render_initialise(self) -> None: def _render_finalise(self) -> None: """Finalise the render of the document.""" + # attempt to replace id_link references with internal links + for refnode in findall(self.document)(nodes.reference): + if not refnode.get("id_link"): + continue + target = refnode["refuri"][1:] + if target in self._slug_to_section: + section_node = self._slug_to_section[target] + refnode["refid"] = section_node["ids"][0] + + if not refnode.children: + implicit_text = clean_astext(section_node[0]) + refnode += nodes.inline( + implicit_text, implicit_text, classes=["std", "std-ref"] + ) + else: + self.create_warning( + f"local id not found: {refnode['refuri']!r}", + MystWarnings.XREF_MISSING, + line=refnode.line, + append_to=refnode, + ) + refnode["refid"] = target + del refnode["refuri"] + + if self._slug_to_section and self.sphinx_env: + # save for later reference resolution + self.sphinx_env.metadata[self.sphinx_env.docname]["myst_slugs"] = { + slug: (snode["ids"][0], clean_astext(snode[0])) + for slug, snode in self._slug_to_section.items() + } + # log warnings for duplicate reference definitions # "duplicate_refs": [{"href": "ijk", "label": "B", "map": [4, 5], "title": ""}], for dup_ref in self.md_env.get("duplicate_refs", []): @@ -713,11 +746,29 @@ def render_heading(self, token: SyntaxTreeNode) -> None: with self.current_node_context(title_node): self.render_children(token) - # create a target reference for the section, based on the heading text + # create a target reference for the section, based on the heading text. + # Note, this is an implicit target, meaning that it is not prioritised, + # and is not stored by sphinx for ref resolution name = nodes.fully_normalize_name(title_node.astext()) new_section["names"].append(name) self.document.note_implicit_target(new_section, new_section) + # add possible reference slug, this may be different to the standard name above, + # and does not have to be normalised, so we treat it separately + if "id" in token.attrs: + slug = str(token.attrs["id"]) + new_section["slug"] = slug + if slug in self._slug_to_section: + other_node = self._slug_to_section[slug] + self.create_warning( + f"duplicate heading slug {slug!r}, other at line {other_node.line}", + MystWarnings.ANCHOR_DUPE, + line=new_section.line, + ) + else: + # we store this for later processing on finalise + self._slug_to_section[slug] = new_section + # set the section as the current node for subsequent rendering self.current_node = new_section @@ -736,19 +787,19 @@ def render_link(self, token: SyntaxTreeNode) -> None: or self.md_config.gfm_only or self.md_config.all_links_external ): - if token.info == "auto": # handles both autolink and linkify - return self.render_autolink(token) - else: - return self.render_external_url(token) + return self.render_external_url(token) href = cast(str, token.attrGet("href") or "") + if href.startswith("#"): + return self.render_id_link(token) + # TODO ideally whether inv_link is enabled could be precomputed if "inv_link" in self.md_config.enable_extensions and href.startswith("inv:"): return self.create_inventory_link(token) if token.info == "auto": # handles both autolink and linkify - return self.render_autolink(token) + return self.render_external_url(token) # Check for external URL url_scheme = urlparse(href).scheme @@ -761,20 +812,27 @@ def render_link(self, token: SyntaxTreeNode) -> None: return self.render_internal_link(token) def render_external_url(self, token: SyntaxTreeNode) -> None: - """Render link token `[text](link "title")`, - where the link has been identified as an external URL:: - - <reference refuri="link" title="title"> - text - - `text` can contain nested syntax, e.g. `[**bold**](url "title")`. + """Render link token (including autolink and linkify), + where the link has been identified as an external URL. """ ref_node = nodes.reference() self.add_line_and_source_path(ref_node, token) self.copy_attributes( token, ref_node, ("class", "id", "reftitle"), aliases={"title": "reftitle"} ) - ref_node["refuri"] = cast(str, token.attrGet("href") or "") + ref_node["refuri"] = escapeHtml(token.attrGet("href") or "") # type: ignore[arg-type] + with self.current_node_context(ref_node, append=True): + self.render_children(token) + + def render_id_link(self, token: SyntaxTreeNode) -> None: + """Render link token like `[text](#id)`, to a local target.""" + ref_node = nodes.reference() + self.add_line_and_source_path(ref_node, token) + ref_node["id_link"] = True + ref_node["refuri"] = token.attrGet("href") or "" + self.copy_attributes( + token, ref_node, ("class", "id", "reftitle"), aliases={"title": "reftitle"} + ) with self.current_node_context(ref_node, append=True): self.render_children(token) @@ -799,17 +857,6 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: with self.current_node_context(ref_node, append=True): self.render_children(token) - def render_autolink(self, token: SyntaxTreeNode) -> None: - refuri = escapeHtml(token.attrGet("href") or "") # type: ignore[arg-type] - ref_node = nodes.reference() - self.copy_attributes( - token, ref_node, ("class", "id", "reftitle"), aliases={"title": "reftitle"} - ) - ref_node["refuri"] = refuri - self.add_line_and_source_path(ref_node, token) - with self.current_node_context(ref_node, append=True): - self.render_children(token) - def create_inventory_link(self, token: SyntaxTreeNode) -> None: r"""Create a link to an inventory object. @@ -1641,3 +1688,15 @@ def html_meta_to_nodes( output.append(pending) return output + + +def clean_astext(node: nodes.Element) -> str: + """Like node.astext(), but ignore images. + Copied from sphinx. + """ + node = node.deepcopy() + for img in findall(node)(nodes.image): + img["alt"] = "" + for raw in list(findall(node)(nodes.raw)): + raw.parent.remove(raw) + return node.astext() diff --git a/myst_parser/mdit_to_docutils/sphinx_.py b/myst_parser/mdit_to_docutils/sphinx_.py index c1989edb..af65c1c7 100644 --- a/myst_parser/mdit_to_docutils/sphinx_.py +++ b/myst_parser/mdit_to_docutils/sphinx_.py @@ -11,15 +11,12 @@ from markdown_it.tree import SyntaxTreeNode from sphinx import addnodes from sphinx.domains.math import MathDomain -from sphinx.domains.std import StandardDomain from sphinx.environment import BuildEnvironment from sphinx.ext.intersphinx import InventoryAdapter from sphinx.util import logging -from sphinx.util.nodes import clean_astext from myst_parser import inventory from myst_parser.mdit_to_docutils.base import DocutilsRenderer -from myst_parser.warnings_ import MystWarnings LOGGER = logging.getLogger(__name__) @@ -49,38 +46,42 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None: destination = os.path.relpath( os.path.join(include_dir, os.path.normpath(destination)), source_dir ) - + kwargs = { + "refdoc": self.sphinx_env.docname, + "reftype": "myst", + "refexplicit": len(token.children or []) > 0, + } + path_dest, *_path_ids = destination.split("#", maxsplit=1) + path_id = _path_ids[0] if _path_ids else None potential_path = ( - Path(self.sphinx_env.doc2path(self.sphinx_env.docname)).parent / destination + Path(self.sphinx_env.doc2path(self.sphinx_env.docname)).parent / path_dest if self.sphinx_env.srcdir # not set in some test situations else None ) - if ( - potential_path - and potential_path.is_file() - and not any( - destination.endswith(suffix) - for suffix in self.sphinx_env.config.source_suffix - ) - ): - wrap_node = addnodes.download_reference( - refdoc=self.sphinx_env.docname, - reftarget=destination, - reftype="myst", - refdomain=None, # Added to enable cross-linking - refexplicit=len(token.children or []) > 0, - refwarn=False, + if path_dest == "./": + # this is a special case, where we want to reference the current document + potential_path = ( + Path(self.sphinx_env.doc2path(self.sphinx_env.docname)) + if self.sphinx_env.srcdir + else None ) - classes = ["xref", "download", "myst"] - text = destination if not token.children else "" + if potential_path and potential_path.is_file(): + docname = self.sphinx_env.path2doc(str(potential_path)) + if docname: + wrap_node = addnodes.pending_xref( + refdomain="doc", reftarget=docname, reftargetid=path_id, **kwargs + ) + classes = ["xref", "myst"] + text = "" + else: + wrap_node = addnodes.download_reference( + refdomain=None, reftarget=path_dest, refwarn=False, **kwargs + ) + classes = ["xref", "download", "myst"] + text = destination if not token.children else "" else: wrap_node = addnodes.pending_xref( - refdoc=self.sphinx_env.docname, - reftarget=destination, - reftype="myst", - refdomain=None, # Added to enable cross-linking - refexplicit=len(token.children or []) > 0, - refwarn=True, + refdomain=None, reftarget=destination, refwarn=True, **kwargs ) classes = ["xref", "myst"] text = "" @@ -112,48 +113,6 @@ def get_inventory_matches( ) ) - def render_heading(self, token: SyntaxTreeNode) -> None: - """This extends the docutils method, to allow for the addition of heading ids. - These ids are computed by the ``markdown-it-py`` ``anchors_plugin`` - as "slugs" which are unique to a document. - - The approach is similar to ``sphinx.ext.autosectionlabel`` - """ - super().render_heading(token) - - if not isinstance(self.current_node, nodes.section): - return - - # create the slug string - slug = cast(str, token.attrGet("id")) - if slug is None: - return - - section = self.current_node - doc_slug = ( - self.sphinx_env.doc2path(self.sphinx_env.docname, base=False) + "#" + slug - ) - - # save the reference in the standard domain, so that it can be handled properly - domain = cast(StandardDomain, self.sphinx_env.get_domain("std")) - if doc_slug in domain.labels: - other_doc = self.sphinx_env.doc2path(domain.labels[doc_slug][0]) - self.create_warning( - f"duplicate label {doc_slug}, other instance in {other_doc}", - MystWarnings.ANCHOR_DUPE, - line=section.line, - ) - labelid = section["ids"][0] - domain.anonlabels[doc_slug] = self.sphinx_env.docname, labelid - domain.labels[doc_slug] = ( - self.sphinx_env.docname, - labelid, - clean_astext(section[0]), - ) - - self.sphinx_env.metadata[self.sphinx_env.docname]["myst_anchors"] = True - section["myst-anchor"] = doc_slug - def render_math_block_label(self, token: SyntaxTreeNode) -> None: """Render math with referencable labels, e.g. ``$a=1$ (label)``.""" label = token.info diff --git a/myst_parser/sphinx_ext/myst_refs.py b/myst_parser/sphinx_ext/myst_refs.py index 948303a3..dc06abfb 100644 --- a/myst_parser/sphinx_ext/myst_refs.py +++ b/myst_parser/sphinx_ext/myst_refs.py @@ -3,7 +3,6 @@ This is applied to MyST type references only, such as ``[text](target)``, and allows for nested syntax """ -import os from typing import Any, List, Optional, Tuple, cast from docutils import nodes @@ -11,6 +10,7 @@ from sphinx import addnodes, version_info from sphinx.addnodes import pending_xref from sphinx.domains.std import StandardDomain +from sphinx.errors import NoUri from sphinx.locale import __ from sphinx.transforms.post_transforms import ReferencesResolver from sphinx.util import docname_join, logging @@ -19,13 +19,12 @@ from myst_parser._compat import findall from myst_parser.warnings_ import MystWarnings -try: - from sphinx.errors import NoUri -except ImportError: - # sphinx < 2.1 - from sphinx.environment import NoUri # type: ignore +LOGGER = logging.getLogger(__name__) -logger = logging.getLogger(__name__) + +def log_warning(msg: str, subtype: MystWarnings, **kwargs: Any): + """Log a warning, with a myst type and specific subtype.""" + LOGGER.warning(msg, type="myst", subtype=subtype.value, **kwargs) class MystReferenceResolver(ReferencesResolver): @@ -42,6 +41,10 @@ def run(self, **kwargs: Any) -> None: if node["reftype"] != "myst": continue + if node["refdomain"] == "doc": + self.resolve_myst_ref_doc(node) + continue + contnode = cast(nodes.TextElement, node[0].deepcopy()) newnode = None @@ -50,7 +53,7 @@ def run(self, **kwargs: Any) -> None: domain = None try: - newnode = self.resolve_myst_ref(refdoc, node, contnode) + newnode = self.resolve_myst_ref_any(refdoc, node, contnode) if newnode is None: # no new node found? try the missing-reference event # but first we change the the reftype to 'any' @@ -84,7 +87,58 @@ def run(self, **kwargs: Any) -> None: node.replace_self(newnode or contnode) - def resolve_myst_ref( + def resolve_myst_ref_doc(self, node: pending_xref): + """Resolve a reference, from a markdown link, to another document, + optionally with a target id within that document. + """ + from_docname = node.get("refdoc", self.env.docname) + ref_docname: str = node["reftarget"] + ref_id: Optional[str] = node["reftargetid"] + + if ref_docname not in self.env.all_docs: + log_warning( + f"Unknown source document {ref_docname!r}", + MystWarnings.XREF_MISSING, + location=node, + ) + node.replace_self(node[0].deepcopy()) + return + + targetid = "" + implicit_text = "" + inner_classes = ["std", "std-doc"] + + if ref_id: + slug_to_section = self.env.metadata[ref_docname].get("myst_slugs", {}) + if ref_id not in slug_to_section: + log_warning( + f"local id not found in doc {ref_docname!r}: {ref_id!r}", + MystWarnings.XREF_MISSING, + location=node, + ) + targetid = ref_id + else: + targetid, implicit_text = slug_to_section[ref_id] + inner_classes = ["std", "std-ref"] + else: + implicit_text = clean_astext(self.env.titles[ref_docname]) + + if node["refexplicit"]: + caption = node.astext() + innernode = nodes.inline(caption, "", classes=inner_classes) + innernode.extend(node[0].children) + else: + innernode = nodes.inline( + implicit_text, implicit_text, classes=inner_classes + ) + + assert self.app.builder + ref_node = make_refnode( + self.app.builder, from_docname, ref_docname, targetid, innernode + ) + node.replace_self(ref_node) + + def resolve_myst_ref_any( self, refdoc: str, node: pending_xref, contnode: Element ) -> Element: """Resolve reference generated by the "myst" role; ``[text](reference)``. @@ -100,22 +154,15 @@ def resolve_myst_ref( target: str = node["reftarget"] results: List[Tuple[str, Element]] = [] - res_anchor = self._resolve_anchor(node, refdoc) - if res_anchor: - results.append(("std:doc", res_anchor)) - else: - # if we've already found an anchored doc, - # don't search in the std:ref/std:doc (leads to duplication) - - # resolve standard references - res = self._resolve_ref_nested(node, refdoc) - if res: - results.append(("std:ref", res)) + # resolve standard references + res = self._resolve_ref_nested(node, refdoc) + if res: + results.append(("std:ref", res)) - # resolve doc names - res = self._resolve_doc_nested(node, refdoc) - if res: - results.append(("std:doc", res)) + # resolve doc names + res = self._resolve_doc_nested(node, refdoc) + if res: + results.append(("std:doc", res)) # get allowed domains for referencing ref_domains = self.env.config.myst_ref_domains @@ -153,11 +200,10 @@ def resolve_myst_ref( # the domain doesn't yet support the new interface # we have to manually collect possible references (SLOW) if not (getattr(domain, "__module__", "").startswith("sphinx.")): - logger.warning( + log_warning( f"Domain '{domain.__module__}::{domain.name}' has not " "implemented a `resolve_any_xref` method [myst.domains]", - type="myst", - subtype=MystWarnings.LEGACY_DOMAIN.value, + MystWarnings.LEGACY_DOMAIN, once=True, ) for role in domain.roles: @@ -177,14 +223,13 @@ def stringify(name, node): return f":{name}:`{reftitle}`" candidates = " or ".join(stringify(name, role) for name, role in results) - logger.warning( + log_warning( __( f"more than one target found for 'myst' cross-reference {target}: " f"could be {candidates} [myst.ref]" ), + MystWarnings.XREF_AMBIGUOUS, location=node, - type="myst", - subtype=MystWarnings.XREF_AMBIGUOUS.value, ) res_role, newnode = results[0] @@ -199,29 +244,6 @@ def stringify(name, node): return newnode - def _resolve_anchor( - self, node: pending_xref, fromdocname: str - ) -> Optional[Element]: - """Resolve doc with anchor.""" - if self.env.config.myst_heading_anchors is None: - # no target anchors will have been created, so we don't look for them - return None - target: str = node["reftarget"] - if "#" not in target: - return None - # the link may be a heading anchor; we need to first get the relative path - rel_path, anchor = target.rsplit("#", 1) - rel_path = os.path.normpath(rel_path) - if rel_path == ".": - # anchor in the same doc as the node - doc_path = self.env.doc2path(node.get("refdoc", fromdocname), base=False) - else: - # anchor in a different doc from the node - doc_path = os.path.normpath( - os.path.join(node.get("refdoc", fromdocname), "..", rel_path) - ) - return self._resolve_ref_nested(node, fromdocname, doc_path + "#" + anchor) - def _resolve_ref_nested( self, node: pending_xref, fromdocname: str, target=None ) -> Optional[Element]: @@ -258,16 +280,9 @@ def _resolve_doc_nested( It also allows for extensions on document names. """ - # directly reference to document by source name; can be absolute or relative - refdoc = node.get("refdoc", fromdocname) - docname = docname_join(refdoc, node["reftarget"]) - + docname = docname_join(node.get("refdoc", fromdocname), node["reftarget"]) if docname not in self.env.all_docs: - # try stripping known extensions from doc name - if os.path.splitext(docname)[1] in self.env.config.source_suffix: - docname = os.path.splitext(docname)[0] - if docname not in self.env.all_docs: - return None + return None if node["refexplicit"]: # reference with explicit title @@ -275,7 +290,6 @@ def _resolve_doc_nested( innernode = nodes.inline(caption, "", classes=["doc"]) innernode.extend(node[0].children) else: - # TODO do we want nested syntax for titles? caption = clean_astext(self.env.titles[docname]) innernode = nodes.inline(caption, caption, classes=["doc"]) diff --git a/myst_parser/warnings_.py b/myst_parser/warnings_.py index 6325c4af..85d18e2d 100644 --- a/myst_parser/warnings_.py +++ b/myst_parser/warnings_.py @@ -32,6 +32,8 @@ class MystWarnings(Enum): # cross-reference resolution XREF_AMBIGUOUS = "xref_ambiguous" """Multiple targets were found for a cross-reference.""" + XREF_MISSING = "xref_missing" + """A target was not found for a cross-reference.""" INV_LOAD = "inv_retrieval" """Failure to retrieve or load an inventory.""" IREF_MISSING = "iref_missing" diff --git a/tests/test_renderers/fixtures/docutil_syntax_elements.md b/tests/test_renderers/fixtures/docutil_syntax_elements.md index 91123a50..bbcbf8bb 100644 --- a/tests/test_renderers/fixtures/docutil_syntax_elements.md +++ b/tests/test_renderers/fixtures/docutil_syntax_elements.md @@ -340,8 +340,11 @@ Title <reference refuri="https://www.google.com"> alt2 <paragraph> - <reference refname="#target3"> + <reference id_link="True" refid="target3"> alt3 + <system_message level="2" line="12" source="notset" type="WARNING"> + <paragraph> + local id not found: '#target3' [myst.xref_missing] . Comments: diff --git a/tests/test_renderers/fixtures/myst-config.txt b/tests/test_renderers/fixtures/myst-config.txt index 0f898cb5..7f997e94 100644 --- a/tests/test_renderers/fixtures/myst-config.txt +++ b/tests/test_renderers/fixtures/myst-config.txt @@ -156,6 +156,20 @@ www.commonmark.org/he<lp <lp . +[heading_anchors] --myst-heading-anchors=1 +. +# My title +[](#my-title) +. +<document ids="my-title" names="my\ title" slug="my-title" source="<string>" title="My title"> + <title> + My title + <paragraph> + <reference id_link="True" refid="my-title"> + <inline classes="std std-ref"> + My title +. + [html_meta] --myst-html-meta='{"keywords": "Sphinx, MyST"}' . text diff --git a/tests/test_renderers/fixtures/sphinx_syntax_elements.md b/tests/test_renderers/fixtures/sphinx_syntax_elements.md index 2d40bdb5..838cf993 100644 --- a/tests/test_renderers/fixtures/sphinx_syntax_elements.md +++ b/tests/test_renderers/fixtures/sphinx_syntax_elements.md @@ -324,7 +324,7 @@ Title [alt2](https://www.google.com) -[alt3](#target3) +[alt3](#title) . <document source="<src>/index.md"> <target ids="target" names="target"> @@ -342,9 +342,11 @@ Title <reference refuri="https://www.google.com"> alt2 <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="True" reftarget="#target3" reftype="myst" refwarn="True"> - <inline classes="xref myst"> - alt3 + <reference id_link="True" refid="title"> + alt3 + <system_message level="2" line="12" source="<src>/index.md" type="WARNING"> + <paragraph> + local id not found: '#title' [myst.xref_missing] . Comments: diff --git a/tests/test_renderers/test_myst_refs/doc_with_extension.xml b/tests/test_renderers/test_myst_refs/doc_with_extension.xml index 55cb74ce..0e8f4678 100644 --- a/tests/test_renderers/test_myst_refs/doc_with_extension.xml +++ b/tests/test_renderers/test_myst_refs/doc_with_extension.xml @@ -1,5 +1,5 @@ <document source="root/index.md"> <paragraph> <reference internal="True" refuri=""> - <inline classes="doc std std-doc"> + <inline classes="std std-doc"> <no title> diff --git a/tests/test_sphinx/test_sphinx_builds/test_heading_slug_func.resolved.xml b/tests/test_sphinx/test_sphinx_builds/test_heading_slug_func.resolved.xml index e48908dd..818a2a7b 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_heading_slug_func.resolved.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_heading_slug_func.resolved.xml @@ -1,7 +1,7 @@ <document source="index.md"> - <section ids="hyphen-1" myst-anchor="index.md#hyphen-1" names="hyphen\ -\ 1"> + <section ids="hyphen-1" names="hyphen\ -\ 1" slug="hyphen-1"> <title> Hyphen - 1 - <section ids="dot-1-1" myst-anchor="index.md#dot-1-1" names="dot\ 1.1"> + <section ids="dot-1-1" names="dot\ 1.1" slug="dot-1-1"> <title> Dot 1.1 diff --git a/tests/test_sphinx/test_sphinx_builds/test_heading_slug_func.xml b/tests/test_sphinx/test_sphinx_builds/test_heading_slug_func.xml index e48908dd..818a2a7b 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_heading_slug_func.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_heading_slug_func.xml @@ -1,7 +1,7 @@ <document source="index.md"> - <section ids="hyphen-1" myst-anchor="index.md#hyphen-1" names="hyphen\ -\ 1"> + <section ids="hyphen-1" names="hyphen\ -\ 1" slug="hyphen-1"> <title> Hyphen - 1 - <section ids="dot-1-1" myst-anchor="index.md#dot-1-1" names="dot\ 1.1"> + <section ids="dot-1-1" names="dot\ 1.1" slug="dot-1-1"> <title> Dot 1.1 diff --git a/tests/test_sphinx/test_sphinx_builds/test_includes.html b/tests/test_sphinx/test_sphinx_builds/test_includes.html index 41eb1ee6..2c1b6d88 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_includes.html +++ b/tests/test_sphinx/test_sphinx_builds/test_includes.html @@ -83,7 +83,7 @@ <h2> </p> <p> <a class="reference internal" href="#"> - <span class="doc std std-doc"> + <span class="std std-doc"> text </span> </a> diff --git a/tests/test_sphinx/test_sphinx_builds/test_includes.xml b/tests/test_sphinx/test_sphinx_builds/test_includes.xml index 66024070..64ff16ab 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_includes.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_includes.xml @@ -37,7 +37,7 @@ <paragraph> <image alt="alt" candidates="{'?': 'https://example.com'}" uri="https://example.com"> <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="True" reftarget="index.md" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="doc" refexplicit="True" reftarget="index" reftargetid="True" reftype="myst"> <inline classes="xref myst"> text <paragraph> diff --git a/tests/test_sphinx/test_sphinx_builds/test_references.html b/tests/test_sphinx/test_sphinx_builds/test_references.html index a6d4036f..63297b93 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_references.html +++ b/tests/test_sphinx/test_sphinx_builds/test_references.html @@ -59,21 +59,21 @@ <h1> </p> <p> <a class="reference internal" href="#"> - <span class="doc std std-doc"> + <span class="std std-doc"> Title with nested a=1 </span> </a> </p> <p> <a class="reference internal" href="#"> - <span class="doc std std-doc"> + <span class="std std-doc"> plain text </span> </a> </p> <p> <a class="reference internal" href="#"> - <span class="doc std std-doc"> + <span class="std std-doc"> nested <em> syntax @@ -155,35 +155,35 @@ <h2> </div> <p> <a class="reference internal" href="#title-anchors"> - <span class="std std-doc"> + <span class="std std-ref"> Title anchors </span> </a> </p> <p> <a class="reference internal" href="#title-anchors"> - <span class="std std-doc"> + <span class="std std-ref"> Title anchors </span> </a> </p> <p> <a class="reference internal" href="other.html#title-anchors"> - <span class="std std-doc"> + <span class="std std-ref"> Title anchors </span> </a> </p> <p> <a class="reference internal" href="other.html#title-anchors"> - <span class="std std-doc"> + <span class="std std-ref"> Title anchors </span> </a> </p> <p> <a class="reference internal" href="subfolder/other2.html#title-anchors"> - <span class="std std-doc"> + <span class="std std-ref"> Title anchors </span> </a> diff --git a/tests/test_sphinx/test_sphinx_builds/test_references.resolved.xml b/tests/test_sphinx/test_sphinx_builds/test_references.resolved.xml index 9c6a4cac..957af06c 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_references.resolved.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_references.resolved.xml @@ -1,6 +1,6 @@ <document source="index.md"> <target refid="title"> - <section classes="tex2jax_ignore mathjax_ignore" ids="title-with-nested-a-1 title" myst-anchor="index.md#title-with-nested" names="title\ with\ nested\ a=1 title"> + <section classes="tex2jax_ignore mathjax_ignore" ids="title-with-nested-a-1 title" names="title\ with\ nested\ a=1 title" slug="title-with-nested"> <title> Title with <strong> @@ -34,15 +34,15 @@ syntax <paragraph> <reference internal="True" refuri=""> - <inline classes="doc std std-doc"> + <inline classes="std std-doc"> Title with nested a=1 <paragraph> <reference internal="True" refuri=""> - <inline classes="doc std std-doc"> + <inline classes="std std-doc"> plain text <paragraph> <reference internal="True" refuri=""> - <inline classes="doc std std-doc"> + <inline classes="std std-doc"> nested <emphasis> syntax @@ -73,7 +73,7 @@ <reference internal="True" refid="insidecodeblock"> <inline classes="std std-ref"> fence - <section ids="title-anchors" myst-anchor="index.md#title-anchors" names="title\ anchors"> + <section ids="title-anchors" names="title\ anchors" slug="title-anchors"> <title> Title <emphasis> @@ -94,22 +94,22 @@ <emphasis> anchors <paragraph> - <reference internal="True" refid="title-anchors"> - <inline classes="std std-doc"> + <reference id_link="True" refid="title-anchors"> + <inline classes="std std-ref"> Title anchors <paragraph> <reference internal="True" refid="title-anchors"> - <inline classes="std std-doc"> + <inline classes="std std-ref"> Title anchors <paragraph> <reference internal="True" refuri="other.html#title-anchors"> - <inline classes="std std-doc"> + <inline classes="std std-ref"> Title anchors <paragraph> <reference internal="True" refuri="other.html#title-anchors"> - <inline classes="std std-doc"> + <inline classes="std std-ref"> Title anchors <paragraph> <reference internal="True" refuri="subfolder/other2.html#title-anchors"> - <inline classes="std std-doc"> + <inline classes="std std-ref"> Title anchors diff --git a/tests/test_sphinx/test_sphinx_builds/test_references.xml b/tests/test_sphinx/test_sphinx_builds/test_references.xml index 03bfb8d5..4cb99ad3 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_references.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_references.xml @@ -1,6 +1,6 @@ <document source="index.md"> <target refid="title"> - <section classes="tex2jax_ignore mathjax_ignore" ids="title-with-nested-a-1 title" myst-anchor="index.md#title-with-nested" names="title\ with\ nested\ a=1 title"> + <section classes="tex2jax_ignore mathjax_ignore" ids="title-with-nested-a-1 title" names="title\ with\ nested\ a=1 title" slug="title-with-nested"> <title> Title with <strong> @@ -32,14 +32,14 @@ <emphasis> syntax <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="False" reftarget="index.md" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="doc" refexplicit="False" reftarget="index" reftargetid="True" reftype="myst"> <inline classes="xref myst"> <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="True" reftarget="index.md" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="doc" refexplicit="True" reftarget="index" reftargetid="True" reftype="myst"> <inline classes="xref myst"> plain text <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="True" reftarget="index.md" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="doc" refexplicit="True" reftarget="index" reftargetid="True" reftype="myst"> <inline classes="xref myst"> nested <emphasis> @@ -71,7 +71,7 @@ <pending_xref refdoc="index" refdomain="True" refexplicit="True" reftarget="insidecodeblock" reftype="myst" refwarn="True"> <inline classes="xref myst"> fence - <section ids="title-anchors" myst-anchor="index.md#title-anchors" names="title\ anchors"> + <section ids="title-anchors" names="title\ anchors" slug="title-anchors"> <title> Title <emphasis> @@ -79,17 +79,18 @@ <compound classes="toctree-wrapper"> <toctree caption="True" entries="(None,\ 'other') (None,\ 'subfolder/other2')" glob="False" hidden="False" includefiles="other subfolder/other2" includehidden="False" maxdepth="-1" numbered="0" parent="index" rawentries="" titlesonly="False"> <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="False" reftarget="#title-anchors" reftype="myst" refwarn="True"> - <inline classes="xref myst"> + <reference id_link="True" refid="title-anchors"> + <inline classes="std std-ref"> + Title anchors <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="False" reftarget="./#title-anchors" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="doc" refexplicit="False" reftarget="index" reftargetid="title-anchors" reftype="myst"> <inline classes="xref myst"> <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="False" reftarget="./other.md#title-anchors" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="doc" refexplicit="False" reftarget="other" reftargetid="title-anchors" reftype="myst"> <inline classes="xref myst"> <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="False" reftarget="other.md#title-anchors" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="doc" refexplicit="False" reftarget="other" reftargetid="title-anchors" reftype="myst"> <inline classes="xref myst"> <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="False" reftarget="subfolder/other2.md#title-anchors" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="doc" refexplicit="False" reftarget="subfolder/other2" reftargetid="title-anchors" reftype="myst"> <inline classes="xref myst"> diff --git a/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.html b/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.html index b3d98a97..643e868d 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.html +++ b/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.html @@ -44,14 +44,14 @@ <h3> </p> <p> <a class="reference internal" href="index.html#document-other/other2"> - <span class="doc std std-doc"> + <span class="std std-doc"> Other 2 Title </span> </a> </p> <p> <a class="reference internal" href="index.html#title"> - <span class="std std-doc"> + <span class="std std-ref"> Title </span> </a> @@ -86,21 +86,21 @@ <h3> </p> <p> <a class="reference internal" href="index.html#document-other/other"> - <span class="doc std std-doc"> + <span class="std std-doc"> Other Title </span> </a> </p> <p> <a class="reference internal" href="#title"> - <span class="std std-doc"> + <span class="std std-ref"> Title </span> </a> </p> <p> <a class="reference internal" href="index.html#other-title"> - <span class="std std-doc"> + <span class="std std-ref"> Other Title </span> </a> diff --git a/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.resolved.xml b/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.resolved.xml index 606e769f..249957fa 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.resolved.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.resolved.xml @@ -1,5 +1,5 @@ <document source="other.md"> - <section ids="other-title" myst-anchor="other/other.md#other-title" names="other\ title"> + <section ids="other-title" names="other\ title" slug="other-title"> <title> Other Title <paragraph> @@ -12,9 +12,9 @@ Other 2 Title <paragraph> <reference internal="True" refuri="index.html#document-other/other2"> - <inline classes="doc std std-doc"> + <inline classes="std std-doc"> Other 2 Title <paragraph> <reference internal="True" refuri="index.html#document-index#title"> - <inline classes="std std-doc"> + <inline classes="std std-ref"> Title diff --git a/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.xml b/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.xml index a209b4f4..b2f0cd62 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_references_singlehtml.xml @@ -1,5 +1,5 @@ <document source="other.md"> - <section ids="other-title" myst-anchor="other/other.md#other-title" names="other\ title"> + <section ids="other-title" names="other\ title" slug="other-title"> <title> Other Title <paragraph> @@ -11,8 +11,8 @@ <literal classes="xref any"> other2 <paragraph> - <pending_xref refdoc="other/other" refdomain="True" refexplicit="False" reftarget="./other2.md" reftype="myst" refwarn="True"> + <pending_xref refdoc="other/other" refdomain="doc" refexplicit="False" reftarget="other/other2" reftargetid="True" reftype="myst"> <inline classes="xref myst"> <paragraph> - <pending_xref refdoc="other/other" refdomain="True" refexplicit="False" reftarget="../index.md#title" reftype="myst" refwarn="True"> + <pending_xref refdoc="other/other" refdomain="doc" refexplicit="False" reftarget="index" reftargetid="title" reftype="myst"> <inline classes="xref myst"> From 01ca355d38e6fdfdbc37178f1dbb3a2eac741d2f Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Wed, 11 Jan 2023 19:43:56 +0100 Subject: [PATCH 17/20] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20Add=20live=20previ?= =?UTF-8?q?ew=20(w/=20pyscript)=20(#679)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/_static/custom.css | 28 ++++++++++ docs/conf.py | 1 + docs/index.md | 13 ++++- docs/live-preview.md | 110 ++++++++++++++++++++++++++++++++++++++++ docs/live_preview.py | 63 +++++++++++++++++++++++ pyproject.toml | 3 +- 6 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 docs/live-preview.md create mode 100644 docs/live_preview.py diff --git a/docs/_static/custom.css b/docs/_static/custom.css index f126fba7..2851864d 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -22,3 +22,31 @@ h3::before { .admonition > .admonition-title, div.admonition.no-icon > .admonition-title { padding-left: .6rem; } + +/* Live preview page */ +iframe.pyscript, textarea.pyscript { + width: 100%; + height: 400px; +} +iframe.pyscript { + padding: 4px; +} +textarea.pyscript { + padding: 30px 20px 20px; + border-radius: 8px; + resize: vertical; + font-size: 16px; + font-family: monospace; +} +.display-flex { + display: flex; +} +.display-inline-block { + display: inline-block; + margin-right: 1rem; + margin-bottom: 0; +} +span.label { + /* pyscript changes this and it messes up footnote labels */ + all: unset; +} diff --git a/docs/conf.py b/docs/conf.py index 09726ff9..6afce085 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,6 +35,7 @@ "sphinxext.rediraffe", "sphinxcontrib.mermaid", "sphinxext.opengraph", + "sphinx_pyscript", ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/index.md b/docs/index.md index 36c0cd05..642b352d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,14 +30,24 @@ sd_hide_title: true A Sphinx and Docutils extension to parse MyST, a rich and extensible flavour of Markdown for authoring technical and scientific documentation. +````{div} sd-d-flex-row ```{button-ref} intro :ref-type: doc :color: primary -:class: sd-rounded-pill +:class: sd-rounded-pill sd-mr-3 Get Started ``` +```{button-ref} live-preview +:ref-type: doc +:color: secondary +:class: sd-rounded-pill + +Live Demo +``` +```` + ::: :::: @@ -115,6 +125,7 @@ The MyST markdown language and MyST parser are both supported by the open commun ```{toctree} :hidden: intro.md +live-preview.md ``` ```{toctree} diff --git a/docs/live-preview.md b/docs/live-preview.md new file mode 100644 index 00000000..095e5340 --- /dev/null +++ b/docs/live-preview.md @@ -0,0 +1,110 @@ +--- +py-config: + splashscreen: + autoclose: true + packages: + - myst-docutils + - docutils==0.19 + - pygments +--- + +# Live Preview + +This is a live preview of the MyST Markdown [docutils renderer](docutils.md). +You can edit the text/configuration below and see the live output.[^note] + +[^note]: Additional styling is usually provided by Sphinx themes. + +```{py-script} +:file: live_preview.py +``` + +::::::::{grid} 1 1 1 2 + +:::::::{grid-item} +:child-align: end + +```{raw} html +<div><u><span id="myst-version"></span></u></div> +``` + +:::::{tab-set} +::::{tab-item} Input text +````{raw} html +<textarea class="pyscript" id="input_myst"> +# Heading 1 + +Hallo world! + +```{note} +An admonition note! +``` + +term +: definition + +$$\pi = 3.14159$$ + +```{list-table} +:header-rows: 1 +:align: center + +* - Header 1 + - Header 2 +* - Item 1 + - Item 2 +``` + +```{figure} https://via.placeholder.com/150 +:width: 100px +:align: center + +Figure caption +``` +</textarea> +```` + +:::: +::::{tab-item} Configuration (YAML) +<textarea class="pyscript" id="input_config"> +# see: https://docutils.sourceforge.io/docs/user/config.html +myst_enable_extensions: +- colon_fence +- deflist +- dollarmath +myst_highlight_code_blocks: false +embed_stylesheet: true +stylesheet_path: +- minimal.css +</textarea> +:::: +::::: + +::::::: +:::::::{grid-item} +:child-align: end + +```{raw} html +<div class="display-flex"> +<label for="output_format" class="display-inline-block">Output Format:</label> +<select id="output_format" class="display-inline-block"> + <option value="pseudoxml">AST</option> + <option value="html5" selected>HTML</option> + <option value="latex">LaTeX</option> +</select> +</div> +``` + +::::{tab-set} +:::{tab-item} HTML Render +<iframe class="pyscript" id="output_html" readonly="true"></iframe> +::: +:::{tab-item} Raw Output +<textarea class="pyscript" id="output_raw" readonly="true"></textarea> +::: +:::{tab-item} Warnings +<textarea class="pyscript" id="output_warnings" readonly="true"></textarea> +::: +:::: +::::::: +:::::::: diff --git a/docs/live_preview.py b/docs/live_preview.py new file mode 100644 index 00000000..474a624f --- /dev/null +++ b/docs/live_preview.py @@ -0,0 +1,63 @@ +from io import StringIO + +import yaml +from docutils.core import publish_string +from js import document + +from myst_parser import __version__ +from myst_parser.parsers.docutils_ import Parser + + +def convert(input_config: str, input_myst: str, writer_name: str) -> dict: + warning_stream = StringIO() + try: + settings = yaml.safe_load(input_config) if input_config else {} + assert isinstance(settings, dict), "not a dictionary" + except Exception as exc: + warning_stream.write(f"ERROR: config load: {exc}\n") + settings = {} + settings.update( + { + "output_encoding": "unicode", + "warning_stream": warning_stream, + } + ) + try: + output = publish_string( + input_myst, + parser=Parser(), + writer_name=writer_name, + settings_overrides=settings, + ) + except Exception as exc: + output = f"ERROR: conversion:\n{exc}" + return {"output": output, "warnings": warning_stream.getvalue()} + + +version_label = document.querySelector("span#myst-version") +config_textarea = document.querySelector("textarea#input_config") +input_textarea = document.querySelector("textarea#input_myst") +output_iframe = document.querySelector("iframe#output_html") +output_raw = document.querySelector("textarea#output_raw") +warnings_textarea = document.querySelector("textarea#output_warnings") +oformat_select = document.querySelector("select#output_format") + + +def do_convert(event=None): + result = convert(config_textarea.value, input_textarea.value, oformat_select.value) + output_raw.value = result["output"] + if "html" in oformat_select.value: + output_iframe.contentDocument.body.innerHTML = result["output"] + else: + output_iframe.contentDocument.body.innerHTML = ( + "Change output format to HTML to see output" + ) + warnings_textarea.value = result["warnings"] + + +version_label.textContent = f"myst-parser v{__version__}" +config_textarea.oninput = do_convert +input_textarea.oninput = do_convert +oformat_select.onchange = do_convert + +do_convert() diff --git a/pyproject.toml b/pyproject.toml index e370148d..bdc67cbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,11 +55,12 @@ linkify = ["linkify-it-py~=1.0"] rtd = [ "ipython", # currently required to get sphinx v5 - "sphinx-book-theme @ git+https://github.com/executablebooks/sphinx-book-theme.git@8da268fce3159755041e8db93e132221a0b0def5#egg=sphinx-book-theme", + "sphinx-book-theme==0.4.0rc1", "sphinx-design", "sphinxext-rediraffe~=0.2.7", "sphinxcontrib.mermaid~=0.7.1", "sphinxext-opengraph~=0.6.3", + "sphinx-pyscript", ] testing = [ "beautifulsoup4", From 505ac7730b44a7080c6360976c88fce6d162292d Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Wed, 11 Jan 2023 19:52:24 +0100 Subject: [PATCH 18/20] =?UTF-8?q?=F0=9F=93=9A=20Bust=20CSS=20cache=20(#681?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/_static/{custom.css => local.css} | 0 docs/conf.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/_static/{custom.css => local.css} (100%) diff --git a/docs/_static/custom.css b/docs/_static/local.css similarity index 100% rename from docs/_static/custom.css rename to docs/_static/local.css diff --git a/docs/conf.py b/docs/conf.py index 6afce085..69b4dae9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -181,7 +181,7 @@ def setup(app: Sphinx): MystWarningsDirective, ) - app.add_css_file("custom.css") + app.add_css_file("local.css") app.add_directive("myst-config", MystConfigDirective) app.add_directive("docutils-cli-help", DocutilsCliHelpDirective) app.add_directive("doc-directive", DirectiveDoc) From 3a7563fe10d7877f72601988325e8b61bd84bde9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Jan 2023 03:23:57 +0100 Subject: [PATCH 19/20] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Update=20pytest=20re?= =?UTF-8?q?quirement=20from=20<7,>=3D6=20to=20>=3D7,<8=20(#674)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Chris Sewell <chrisj_sewell@hotmail.com> --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bdc67cbc..b2afeb6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ rtd = [ testing = [ "beautifulsoup4", "coverage[toml]", - "pytest>=6,<7", + "pytest>=7,<8", "pytest-cov", "pytest-regressions", "pytest-param-files~=0.3.4", @@ -73,7 +73,7 @@ testing = [ ] testing-docutils = [ "pygments", - "pytest>=6,<7", + "pytest>=7,<8", "pytest-param-files~=0.3.4", ] From dff96c4f524ad38ccd95d7962210eee372f04c04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Jan 2023 03:29:28 +0100 Subject: [PATCH 20/20] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Update=20sphinxext-o?= =?UTF-8?q?pengraph=20requirement=20from=20~=3D0.6.3=20to=20~=3D0.7.5=20(#?= =?UTF-8?q?676)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b2afeb6e..7ba04f19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ rtd = [ "sphinx-design", "sphinxext-rediraffe~=0.2.7", "sphinxcontrib.mermaid~=0.7.1", - "sphinxext-opengraph~=0.6.3", + "sphinxext-opengraph~=0.7.5", "sphinx-pyscript", ] testing = [