fix: cache markdown it instance and properly dispose shiki on config reload (#4321)

This results in over 5x speedup in build times of certain projects. But this comes at the cost of correctness. `createMarkdownRenderer` now ignores any arguments passed by user. But from our GitHub code search we didn't find any user passing options different than their siteConfig to this function. If you are doing that, please open an issue and we can discuss the best way forward.

---
Co-authored-by: Divyansh Singh <40380293+brc-dd@users.noreply.github.com>

Author: Barbapapazes

package.json

@@ -100,20 +100,20 @@
   "dependencies": {
     "@docsearch/css": "^3.6.2",
     "@docsearch/js": "^3.6.2",
-    "@shikijs/core": "^1.22.0",
-    "@shikijs/transformers": "^1.22.0",
-    "@shikijs/types": "^1.22.0",
+    "@shikijs/core": "^1.22.2",
+    "@shikijs/transformers": "^1.22.2",
+    "@shikijs/types": "^1.22.2",
     "@types/markdown-it": "^14.1.2",
     "@vitejs/plugin-vue": "^5.1.4",
-    "@vue/devtools-api": "^7.4.6",
+    "@vue/devtools-api": "^7.5.4",
     "@vue/shared": "^3.5.12",
     "@vueuse/core": "^11.1.0",
     "@vueuse/integrations": "^11.1.0",
     "focus-trap": "^7.6.0",
     "mark.js": "8.11.1",
     "minisearch": "^7.1.0",
-    "shiki": "^1.22.0",
-    "vite": "^5.4.8",
+    "shiki": "^1.22.2",
+    "vite": "^5.4.10",
     "vue": "^3.5.12"
   },
   "devDependencies": {
@@ -127,7 +127,7 @@
     "@mdit-vue/shared": "^2.1.3",
     "@polka/compression": "^1.0.0-next.28",
     "@rollup/plugin-alias": "^5.1.1",
-    "@rollup/plugin-commonjs": "^28.0.0",
+    "@rollup/plugin-commonjs": "^28.0.1",
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^15.3.0",
     "@rollup/plugin-replace": "^6.0.1",
@@ -141,15 +141,15 @@
     "@types/markdown-it-emoji": "^3.0.1",
     "@types/micromatch": "^4.0.9",
     "@types/minimist": "^1.2.5",
-    "@types/node": "^22.7.5",
+    "@types/node": "^22.8.2",
     "@types/postcss-prefix-selector": "^1.16.3",
     "@types/prompts": "^2.4.9",
     "chokidar": "^3.6.0",
     "conventional-changelog-cli": "^5.0.0",
     "cross-spawn": "^7.0.3",
     "debug": "^4.3.7",
     "esbuild": "^0.24.0",
-    "execa": "^9.4.0",
+    "execa": "^9.5.1",
     "fs-extra": "^11.2.0",
     "get-port": "^7.1.0",
     "gray-matter": "^4.0.3",
@@ -164,31 +164,31 @@
     "markdown-it-mathjax3": "^4.3.2",
     "micromatch": "^4.0.8",
     "minimist": "^1.2.8",
-    "nanoid": "^5.0.7",
+    "nanoid": "^5.0.8",
     "ora": "^8.1.0",
     "p-map": "^7.0.2",
     "path-to-regexp": "^6.3.0",
-    "picocolors": "^1.1.0",
+    "picocolors": "^1.1.1",
     "pkg-dir": "^8.0.0",
-    "playwright-chromium": "^1.48.0",
+    "playwright-chromium": "^1.48.2",
     "polka": "^1.0.0-next.28",
     "postcss-prefix-selector": "^2.1.0",
     "prettier": "^3.3.3",
     "prompts": "^2.4.2",
     "punycode": "^2.3.1",
     "rimraf": "^6.0.1",
-    "rollup": "^4.24.0",
+    "rollup": "^4.24.2",
     "rollup-plugin-dts": "^6.1.1",
     "rollup-plugin-esbuild": "^6.1.1",
     "semver": "^7.6.3",
     "simple-git-hooks": "^2.11.1",
     "sirv": "^3.0.0",
     "sitemap": "^8.0.0",
     "supports-color": "^9.4.0",
-    "tinyglobby": "^0.2.9",
+    "tinyglobby": "^0.2.10",
     "typescript": "^5.6.3",
-    "vitest": "^2.1.2",
-    "vue-tsc": "^2.1.6",
+    "vitest": "^2.1.4",
+    "vue-tsc": "^2.1.8",
     "wait-on": "^8.0.1"
   },
   "peerDependencies": {
@@ -203,7 +203,7 @@
       "optional": true
     }
   },
-  "packageManager": "pnpm@9.12.1",
+  "packageManager": "pnpm@9.12.3",
   "pnpm": {
     "peerDependencyRules": {
       "ignoreMissing": [

pnpm-lock.yaml

@@ -4,10 +4,7 @@ import { glob, type GlobOptions } from 'tinyglobby'
 import type { SiteConfig } from './config'
 import matter from 'gray-matter'
 import { normalizePath } from 'vite'
-import {
-  createMarkdownRenderer,
-  type MarkdownRenderer
-} from './markdown/markdown'
+import { createMarkdownRenderer } from './markdown/markdown'
 
 export interface ContentOptions<T = ContentData[]> {
   /**
@@ -100,15 +97,7 @@ export function createContentLoader<T = ContentData[]>(
   if (typeof pattern === 'string') pattern = [pattern]
   pattern = pattern.map((p) => normalizePath(path.join(config.srcDir, p)))
 
-  let md: MarkdownRenderer
-
-  const cache = new Map<
-    string,
-    {
-      data: any
-      timestamp: number
-    }
-  >()
+  const cache = new Map<string, { data: any; timestamp: number }>()
 
   return {
     watch: pattern,
@@ -124,14 +113,12 @@ export function createContentLoader<T = ContentData[]>(
         ).sort()
       }
 
-      md =
-        md ||
-        (await createMarkdownRenderer(
-          config.srcDir,
-          config.markdown,
-          config.site.base,
-          config.logger
-        ))
+      const md = await createMarkdownRenderer(
+        config.srcDir,
+        config.markdown,
+        config.site.base,
+        config.logger
+      )
 
       const raw: ContentData[] = []
 

src/node/contentLoader.ts

@@ -28,7 +28,7 @@ import type {
 import type { Logger } from 'vite'
 import { containerPlugin, type ContainerOptions } from './plugins/containers'
 import { gitHubAlertsPlugin } from './plugins/githubAlerts'
-import { highlight } from './plugins/highlight'
+import { highlight as createHighlighter } from './plugins/highlight'
 import { highlightLinePlugin } from './plugins/highlightLines'
 import { imagePlugin, type Options as ImageOptions } from './plugins/image'
 import { lineNumberPlugin } from './plugins/lineNumbers'
@@ -173,7 +173,7 @@ export interface MarkdownOptions extends Options {
    */
   container?: ContainerOptions
   /**
-   * Math support (experimental)
+   * Math support
    *
    * You need to install `markdown-it-mathjax3` and set `math` to `true` to enable it.
    * You can also pass options to `markdown-it-mathjax3` here.
@@ -192,22 +192,38 @@ export interface MarkdownOptions extends Options {
 
 export type MarkdownRenderer = MarkdownIt
 
-export const createMarkdownRenderer = async (
+let md: MarkdownRenderer | undefined
+let _disposeHighlighter: (() => void) | undefined
+
+export function disposeMdItInstance() {
+  if (md) {
+    md = undefined
+    _disposeHighlighter?.()
+  }
+}
+
+/**
+ * @experimental
+ */
+export async function createMarkdownRenderer(
   srcDir: string,
   options: MarkdownOptions = {},
   base = '/',
   logger: Pick<Logger, 'warn'> = console
-): Promise<MarkdownRenderer> => {
+): Promise<MarkdownRenderer> {
+  if (md) return md
+
   const theme = options.theme ?? { light: 'github-light', dark: 'github-dark' }
   const codeCopyButtonTitle = options.codeCopyButtonTitle || 'Copy Code'
   const hasSingleTheme = typeof theme === 'string' || 'name' in theme
 
-  const md = MarkdownIt({
-    html: true,
-    linkify: true,
-    highlight: options.highlight || (await highlight(theme, options, logger)),
-    ...options
-  })
+  let [highlight, dispose] = options.highlight
+    ? [options.highlight, () => {}]
+    : await createHighlighter(theme, options, logger)
+
+  _disposeHighlighter = dispose
+
+  md = MarkdownIt({ html: true, linkify: true, highlight, ...options })
 
   md.linkify.set({ fuzzyLink: false })
   md.use(restoreEntities)

src/node/markdown/markdown.ts

@@ -23,7 +23,7 @@ const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)
  * 2. convert line numbers into line options:
  *    [{ line: number, classes: string[] }]
  */
-const attrsToLines = (attrs: string): TransformerCompactLineOption[] => {
+function attrsToLines(attrs: string): TransformerCompactLineOption[] {
   attrs = attrs.replace(/^(?:\[.*?\])?.*?([\d,-]+).*/, '$1').trim()
   const result: number[] = []
   if (!attrs) {
@@ -51,7 +51,7 @@ export async function highlight(
   theme: ThemeOptions,
   options: MarkdownOptions,
   logger: Pick<Logger, 'warn'> = console
-): Promise<(str: string, lang: string, attrs: string) => string> {
+): Promise<[(str: string, lang: string, attrs: string) => string, () => void]> {
   const {
     defaultHighlightLang: defaultLang = '',
     codeTransformers: userTransformers = []
@@ -95,93 +95,96 @@ export async function highlight(
   const lineNoRE = /:(no-)?line-numbers(=\d*)?$/
   const mustacheRE = /\{\{.*?\}\}/g
 
-  return (str: string, lang: string, attrs: string) => {
-    const vPre = vueRE.test(lang) ? '' : 'v-pre'
-    lang =
-      lang
-        .replace(lineNoStartRE, '')
-        .replace(lineNoRE, '')
-        .replace(vueRE, '')
-        .toLowerCase() || defaultLang
+  return [
+    (str: string, lang: string, attrs: string) => {
+      const vPre = vueRE.test(lang) ? '' : 'v-pre'
+      lang =
+        lang
+          .replace(lineNoStartRE, '')
+          .replace(lineNoRE, '')
+          .replace(vueRE, '')
+          .toLowerCase() || defaultLang
 
-    if (lang) {
-      const langLoaded = highlighter.getLoadedLanguages().includes(lang as any)
-      if (!langLoaded && !isSpecialLang(lang)) {
-        logger.warn(
-          c.yellow(
-            `\nThe language '${lang}' is not loaded, falling back to '${
-              defaultLang || 'txt'
-            }' for syntax highlighting.`
+      if (lang) {
+        const langLoaded = highlighter.getLoadedLanguages().includes(lang)
+        if (!langLoaded && !isSpecialLang(lang)) {
+          logger.warn(
+            c.yellow(
+              `\nThe language '${lang}' is not loaded, falling back to '${
+                defaultLang || 'txt'
+              }' for syntax highlighting.`
+            )
           )
-        )
-        lang = defaultLang
+          lang = defaultLang
+        }
       }
-    }
 
-    const lineOptions = attrsToLines(attrs)
-    const mustaches = new Map<string, string>()
+      const lineOptions = attrsToLines(attrs)
+      const mustaches = new Map<string, string>()
 
-    const removeMustache = (s: string) => {
-      if (vPre) return s
-      return s.replace(mustacheRE, (match) => {
-        let marker = mustaches.get(match)
-        if (!marker) {
-          marker = nanoid()
-          mustaches.set(match, marker)
-        }
-        return marker
-      })
-    }
+      const removeMustache = (s: string) => {
+        if (vPre) return s
+        return s.replace(mustacheRE, (match) => {
+          let marker = mustaches.get(match)
+          if (!marker) {
+            marker = nanoid()
+            mustaches.set(match, marker)
+          }
+          return marker
+        })
+      }
 
-    const restoreMustache = (s: string) => {
-      mustaches.forEach((marker, match) => {
-        s = s.replaceAll(marker, match)
-      })
-      return s
-    }
+      const restoreMustache = (s: string) => {
+        mustaches.forEach((marker, match) => {
+          s = s.replaceAll(marker, match)
+        })
+        return s
+      }
 
-    str = removeMustache(str).trimEnd()
+      str = removeMustache(str).trimEnd()
 
-    const highlighted = highlighter.codeToHtml(str, {
-      lang,
-      transformers: [
-        ...transformers,
-        transformerCompactLineOptions(lineOptions),
-        {
-          name: 'vitepress:v-pre',
-          pre(node) {
-            if (vPre) node.properties['v-pre'] = ''
-          }
-        },
-        {
-          name: 'vitepress:empty-line',
-          code(hast) {
-            hast.children.forEach((span) => {
-              if (
-                span.type === 'element' &&
-                span.tagName === 'span' &&
-                Array.isArray(span.properties.class) &&
-                span.properties.class.includes('line') &&
-                span.children.length === 0
-              ) {
-                span.children.push({
-                  type: 'element',
-                  tagName: 'wbr',
-                  properties: {},
-                  children: []
-                })
-              }
-            })
-          }
-        },
-        ...userTransformers
-      ],
-      meta: { __raw: attrs },
-      ...(typeof theme === 'object' && 'light' in theme && 'dark' in theme
-        ? { themes: theme, defaultColor: false }
-        : { theme })
-    })
+      const highlighted = highlighter.codeToHtml(str, {
+        lang,
+        transformers: [
+          ...transformers,
+          transformerCompactLineOptions(lineOptions),
+          {
+            name: 'vitepress:v-pre',
+            pre(node) {
+              if (vPre) node.properties['v-pre'] = ''
+            }
+          },
+          {
+            name: 'vitepress:empty-line',
+            code(hast) {
+              hast.children.forEach((span) => {
+                if (
+                  span.type === 'element' &&
+                  span.tagName === 'span' &&
+                  Array.isArray(span.properties.class) &&
+                  span.properties.class.includes('line') &&
+                  span.children.length === 0
+                ) {
+                  span.children.push({
+                    type: 'element',
+                    tagName: 'wbr',
+                    properties: {},
+                    children: []
+                  })
+                }
+              })
+            }
+          },
+          ...userTransformers
+        ],
+        meta: { __raw: attrs },
+        ...(typeof theme === 'object' && 'light' in theme && 'dark' in theme
+          ? { themes: theme, defaultColor: false }
+          : { theme })
+      })
 
-    return restoreMustache(highlighted)
-  }
+      return restoreMustache(highlighted)
+    },
+    highlighter.dispose
+  ]
 }

src/node/markdown/plugins/highlight.ts

@@ -16,6 +16,7 @@ import {
   resolveAliases
 } from './alias'
 import { resolvePages, resolveUserConfig, type SiteConfig } from './config'
+import { disposeMdItInstance } from './markdown/markdown'
 import {
   clearCache,
   createMarkdownToVueRenderFn,
@@ -388,6 +389,7 @@ export async function createVitePressPlugin(
           return
         }
 
+        disposeMdItInstance()
         clearCache()
         await recreateServer?.()
         return