feat: migrate to shikiji (#3237)

Author: antfu

__tests__/e2e/markdown-extensions/markdown-extensions.test.ts

@@ -163,19 +163,19 @@ describe('Line Numbers', () => {
 describe('Import Code Snippets', () => {
   test('basic', async () => {
     const lines = page.locator('#basic-code-snippet + div code > span')
-    expect(await lines.count()).toBe(22)
+    expect(await lines.count()).toBe(11)
   })
 
   test('specify region', async () => {
     const lines = page.locator('#specify-region + div code > span')
-    expect(await lines.count()).toBe(6)
+    expect(await lines.count()).toBe(3)
   })
 
   test('with other features', async () => {
     const div = page.locator('#with-other-features + div')
     expect(await getClassList(div)).toContain('line-numbers-mode')
     const lines = div.locator('code > span')
-    expect(await lines.count()).toBe(6)
+    expect(await lines.count()).toBe(3)
     expect(await getClassList(lines.nth(0))).toContain('highlighted')
   })
 })
@@ -216,10 +216,10 @@ describe('Code Groups', () => {
 
     // blocks
     const blocks = div.locator('.blocks > div')
-    expect(await blocks.nth(0).locator('code > span').count()).toBe(22)
+    expect(await blocks.nth(0).locator('code > span').count()).toBe(11)
     expect(await getClassList(blocks.nth(1))).toContain('line-numbers-mode')
     expect(await getClassList(blocks.nth(1))).toContain('language-ts')
-    expect(await blocks.nth(1).locator('code > span').count()).toBe(6)
+    expect(await blocks.nth(1).locator('code > span').count()).toBe(3)
     expect(
       await getClassList(blocks.nth(1).locator('code > span').nth(0))
     ).toContain('highlighted')

docs/.vitepress/config.ts

@@ -13,7 +13,15 @@ export default defineConfig({
   cleanUrls: true,
 
   markdown: {
-    math: true
+    math: true,
+    codeTransformers: [
+      // We use `[!!code` in demo to prevent transformation, here we revert it back.
+      {
+        postprocess(code) {
+          return code.replace(/\[\!\!code/g, '[!code')
+        }
+      }
+    ]
   },
 
   sitemap: {

docs/guide/markdown.md

@@ -80,7 +80,7 @@ For more details, see [Frontmatter](../reference/frontmatter-config).
 
 **Input**
 
-```
+```md
 | Tables        |      Are      |  Cool |
 | ------------- | :-----------: | ----: |
 | col 3 is      | right-aligned | $1600 |
@@ -265,7 +265,7 @@ Wraps in a <div class="vp-raw">
 
 ## Syntax Highlighting in Code Blocks
 
-VitePress uses [Shiki](https://shiki.matsu.io/) to highlight language syntax in Markdown code blocks, using coloured text. Shiki supports a wide variety of programming languages. All you need to do is append a valid language alias to the beginning backticks for the code block:
+VitePress uses [Shikiji](https://github.com/antfu/shikiji) (an improved version of [Shiki](https://shiki.matsu.io/)) to highlight language syntax in Markdown code blocks, using coloured text. Shiki supports a wide variety of programming languages. All you need to do is append a valid language alias to the beginning backticks for the code block:
 
 **Input**
 
@@ -377,7 +377,7 @@ export default { // Highlighted
 }
 ```
 
-Alternatively, it's possible to highlight directly in the line by using the `// [!code hl]` comment.
+Alternatively, it's possible to highlight directly in the line by using the `// [!code hightlight]` comment.
 
 **Input**
 
@@ -386,7 +386,7 @@ Alternatively, it's possible to highlight directly in the line by using the `//
 export default {
   data () {
     return {
-      msg: 'Highlighted!' // [!code  hl]
+      msg: 'Highlighted!' // [!!code highlight]
     }
   }
 }
@@ -399,7 +399,7 @@ export default {
 export default {
   data() {
     return {
-      msg: 'Highlighted!' // [!code hl]
+      msg: 'Highlighted!' // [!code highlight]
     }
   }
 }
@@ -413,14 +413,12 @@ Additionally, you can define a number of lines to focus using `// [!code focus:<
 
 **Input**
 
-Note that only one space is required after `!code`, here are two to prevent processing.
-
 ````
 ```js
 export default {
   data () {
     return {
-      msg: 'Focused!' // [!code  focus]
+      msg: 'Focused!' // [!!code focus]
     }
   }
 }
@@ -445,15 +443,13 @@ Adding the `// [!code --]` or `// [!code ++]` comments on a line will create a d
 
 **Input**
 
-Note that only one space is required after `!code`, here are two to prevent processing.
-
 ````
 ```js
 export default {
   data () {
     return {
-      msg: 'Removed' // [!code  --]
-      msg: 'Added' // [!code  ++]
+      msg: 'Removed' // [!!code --]
+      msg: 'Added' // [!!code ++]
     }
   }
 }
@@ -479,15 +475,13 @@ Adding the `// [!code warning]` or `// [!code error]` comments on a line will co
 
 **Input**
 
-Note that only one space is required after `!code`, here are two to prevent processing.
-
 ````
 ```js
 export default {
   data () {
     return {
-      msg: 'Error', // [!code  error]
-      msg: 'Warning' // [!code  warning]
+      msg: 'Error', // [!!code error]
+      msg: 'Warning' // [!!code warning]
     }
   }
 }

docs/reference/site-config.md

@@ -454,95 +454,15 @@ When using the default theme, enabling this option will display each page's last
 
 - Type: `MarkdownOption`
 
-Configure Markdown parser options. VitePress uses [Markdown-it](https://github.com/markdown-it/markdown-it) as the parser, and [Shiki](https://shiki.matsu.io/) to highlight language syntax. Inside this option, you may pass various Markdown related options to fit your needs.
+Configure Markdown parser options. VitePress uses [Markdown-it](https://github.com/markdown-it/markdown-it) as the parser, and [Shikiji](https://github.com/antfu/shikiji) (an improved version of [Shiki](https://shiki.matsu.io/)) to highlight language syntax. Inside this option, you may pass various Markdown related options to fit your needs.
 
 ```js
 export default {
   markdown: {...}
 }
 ```
 
-Below are all the options that you can have in this object:
-
-```ts
-interface MarkdownOptions extends MarkdownIt.Options {
-  // Custom theme for syntax highlighting.
-  // You can use an existing theme.
-  // See: https://github.com/shikijs/shiki/blob/main/docs/themes.md#all-themes
-  // Or add your own theme.
-  // See: https://github.com/shikijs/shiki/blob/main/docs/themes.md#loading-theme
-  theme?:
-    | Shiki.IThemeRegistration
-    | { light: Shiki.IThemeRegistration; dark: Shiki.IThemeRegistration }
-
-  // Enable line numbers in code block.
-  lineNumbers?: boolean
-
-  // Add support for your own languages.
-  // https://github.com/shikijs/shiki/blob/main/docs/languages.md#supporting-your-own-languages-with-shiki
-  languages?: Shiki.ILanguageRegistration[]
-
-  // markdown-it-anchor plugin options.
-  // See: https://github.com/valeriangalliat/markdown-it-anchor#usage
-  anchor?: anchorPlugin.AnchorOptions
-
-  // markdown-it-attrs plugin options.
-  // See: https://github.com/arve0/markdown-it-attrs
-  attrs?: {
-    leftDelimiter?: string
-    rightDelimiter?: string
-    allowedAttributes?: Array<string | RegExp>
-    disable?: boolean
-  }
-
-  // specify default language for syntax highlighter
-  defaultHighlightLang?: string
-
-  // @mdit-vue/plugin-frontmatter plugin options.
-  // See: https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-frontmatter#options
-  frontmatter?: FrontmatterPluginOptions
-
-  // @mdit-vue/plugin-headers plugin options.
-  // See: https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-headers#options
-  headers?: HeadersPluginOptions | boolean
-
-  // @mdit-vue/plugin-sfc plugin options.
-  // See: https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-sfc#options
-  sfc?: SfcPluginOptions
-
-  // @mdit-vue/plugin-toc plugin options.
-  // See: https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-toc#options
-  toc?: TocPluginOptions
-
-  // @mdit-vue/plugin-component plugin options.
-  // See: https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-component#options
-  component?: ComponentPluginOptions
-
-  // Configure the Markdown-it instance.
-  config?: (md: MarkdownIt) => void
-
-  // Same as `config` but will be applied before all other plugins.
-  preConfig?: (md: MarkdownIt) => void
-
-  // Disable cache (experimental)
-  cache?: boolean
-
-  // Math support (experimental)
-  // You need to install `markdown-it-mathjax3` and set `math` to `true` to enable it.
-  // You can also pass options to `markdown-it-mathjax3` here.
-  // See: https://github.com/tani/markdown-it-mathjax3#customization
-  math?: boolean | any
-
-  // Global custom container titles
-  container?: {
-    infoLabel?: string
-    tipLabel?: string
-    warningLabel?: string
-    dangerLabel?: string
-    detailsLabel?: string
-  }
-}
-```
+Check the [type declaration and jsdocs](https://github.com/vuejs/vitepress/blob/main/src/node/markdown/markdown.ts) for all the options available.
 
 ### vite
 

package.json

@@ -101,7 +101,8 @@
     "mark.js": "8.11.1",
     "minisearch": "^6.2.0",
     "mrmime": "^1.0.1",
-    "shiki": "^0.14.5",
+    "shikiji": "^0.7.2",
+    "shikiji-transformers": "^0.7.2",
     "vite": "^5.0.0",
     "vue": "^3.3.8"
   },
@@ -187,7 +188,6 @@
     "rollup-plugin-dts": "^6.1.0",
     "rollup-plugin-esbuild": "^6.1.0",
     "semver": "^7.5.4",
-    "shiki-processor": "^0.1.3",
     "simple-git-hooks": "^2.9.0",
     "sirv": "^2.0.3",
     "sitemap": "^7.1.1",

pnpm-lock.yaml

@@ -1,7 +1,7 @@
-.dark .vp-code-light {
-  display: none;
+.dark .vp-code span {
+  color: var(--shiki-dark, inherit);
 }
 
-html:not(.dark) .vp-code-dark {
-  display: none;
+html:not(.dark) .vp-code span {
+  color: var(--shiki-light, inherit);
 }

src/client/theme-default/styles/components/vp-code.css

@@ -18,7 +18,6 @@ import MarkdownIt from 'markdown-it'
 import anchorPlugin from 'markdown-it-anchor'
 import attrsPlugin from 'markdown-it-attrs'
 import emojiPlugin from 'markdown-it-emoji'
-import type { ILanguageRegistration, IThemeRegistration } from 'shiki'
 import type { Logger } from 'vite'
 import { containerPlugin, type ContainerOptions } from './plugins/containers'
 import { highlight } from './plugins/highlight'
@@ -28,36 +27,139 @@ import { lineNumberPlugin } from './plugins/lineNumbers'
 import { linkPlugin } from './plugins/link'
 import { preWrapperPlugin } from './plugins/preWrapper'
 import { snippetPlugin } from './plugins/snippet'
+import type {
+  ThemeRegistration,
+  BuiltinTheme,
+  LanguageInput,
+  ShikijiTransformer
+} from 'shikiji'
 
 export type { Header } from '../shared'
 
 export type ThemeOptions =
-  | IThemeRegistration
-  | { light: IThemeRegistration; dark: IThemeRegistration }
+  | ThemeRegistration
+  | BuiltinTheme
+  | {
+      light: ThemeRegistration | BuiltinTheme
+      dark: ThemeRegistration | BuiltinTheme
+    }
 
 export interface MarkdownOptions extends MarkdownIt.Options {
-  lineNumbers?: boolean
+  /* ==================== General Options ==================== */
+
+  /**
+   * Setup markdown-it instance before applying plugins
+   */
   preConfig?: (md: MarkdownIt) => void
+  /**
+   * Setup markdown-it instance
+   */
   config?: (md: MarkdownIt) => void
+  /**
+   * Disable cache (experimental)
+   */
+  cache?: boolean
+  externalLinks?: Record<string, string>
+
+  /* ==================== Syntax Highlighting ==================== */
+
+  /**
+   * Custom theme for syntax highlighting.
+   *
+   * You can also pass an object with `light` and `dark` themes to support dual themes.
+   *
+   * @example { theme: 'github-dark' }
+   * @example { theme: { light: 'github-light', dark: 'github-dark' } }
+   *
+   * You can use an existing theme.
+   * @see https://github.com/antfu/shikiji/blob/main/docs/themes.md#all-themes
+   * Or add your own theme.
+   * @see https://github.com/antfu/shikiji/blob/main/docs/themes.md#load-custom-themes
+   */
+  theme?: ThemeOptions
+  /**
+   * Languages for syntax highlighting.
+   * @see https://github.com/antfu/shikiji/blob/main/docs/languages.md#all-themes
+   */
+  languages?: LanguageInput[]
+  /**
+   * Custom language aliases.
+   *
+   * @example { 'my-lang': 'js' }
+   * @see https://github.com/antfu/shikiji/tree/main#custom-language-aliases
+   */
+  languageAlias?: Record<string, string>
+  /**
+   * Show line numbers in code blocks
+   * @default false
+   */
+  lineNumbers?: boolean
+  /**
+   * Fallback language when the specified language is not available.
+   */
+  defaultHighlightLang?: string
+  /**
+   * Transformers applied to code blocks
+   * @see https://github.com/antfu/shikiji#hast-transformers
+   */
+  codeTransformers?: ShikijiTransformer[]
+
+  /* ==================== Markdown It Plugins ==================== */
+
+  /**
+   * Options for `markdown-it-anchor`
+   * @see https://github.com/valeriangalliat/markdown-it-anchor
+   */
   anchor?: anchorPlugin.AnchorOptions
+  /**
+   * Options for `markdown-it-attrs`
+   * @see https://github.com/arve0/markdown-it-attrs
+   */
   attrs?: {
     leftDelimiter?: string
     rightDelimiter?: string
     allowedAttributes?: Array<string | RegExp>
     disable?: boolean
   }
-  defaultHighlightLang?: string
+  /**
+   * Options for `@mdit-vue/plugin-frontmatter`
+   * @see https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-frontmatter
+   */
   frontmatter?: FrontmatterPluginOptions
+  /**
+   * Options for `@mdit-vue/plugin-headers`
+   * @see https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-headers
+   */
   headers?: HeadersPluginOptions | boolean
+  /**
+   * Options for `@mdit-vue/plugin-sfc`
+   * @see https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-sfc
+   */
   sfc?: SfcPluginOptions
-  theme?: ThemeOptions
-  languages?: ILanguageRegistration[]
+  /**
+   * Options for `@mdit-vue/plugin-toc`
+   * @see https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-toc
+   */
   toc?: TocPluginOptions
-  externalLinks?: Record<string, string>
-  cache?: boolean
+  /**
+   * Options for `@mdit-vue/plugin-component`
+   * @see https://github.com/mdit-vue/mdit-vue/tree/main/packages/plugin-component
+   */
   component?: ComponentPluginOptions
-  math?: boolean | any
+  /**
+   * Options for `markdown-it-container`
+   * @see https://github.com/markdown-it/markdown-it-container
+   */
   container?: ContainerOptions
+  /**
+   * Math support (experimental)
+   *
+   * You need to install `markdown-it-mathjax3` and set `math` to `true` to enable it.
+   * You can also pass options to `markdown-it-mathjax3` here.
+   * @default false
+   * @see https://vitepress.dev/guide/markdown#math-equations
+   */
+  math?: boolean | any
 }
 
 export type MarkdownRenderer = MarkdownIt
@@ -80,7 +182,9 @@ export const createMarkdownRenderer = async (
         theme,
         options.languages,
         options.defaultHighlightLang,
-        logger
+        logger,
+        options.codeTransformers,
+        options.languageAlias
       )),
     ...options
   })

src/node/markdown/markdown.ts

@@ -1,23 +1,23 @@
 import { customAlphabet } from 'nanoid'
 import c from 'picocolors'
+import type { LanguageInput, ShikijiTransformer } from 'shikiji'
 import {
-  BUNDLED_LANGUAGES,
-  type HtmlRendererOptions,
-  type ILanguageRegistration,
-  type IThemeRegistration
-} from 'shiki'
-import {
-  addClass,
-  createDiffProcessor,
-  createFocusProcessor,
-  createHighlightProcessor,
-  createRangeProcessor,
-  defineProcessor,
+  bundledLanguages,
   getHighlighter,
-  type Processor
-} from 'shiki-processor'
+  addClassToHast,
+  isPlaintext as isPlainLang,
+  isSpecialLang
+} from 'shikiji'
 import type { Logger } from 'vite'
 import type { ThemeOptions } from '../markdown'
+import {
+  transformerCompactLineOptions,
+  transformerNotationDiff,
+  transformerNotationErrorLevel,
+  transformerNotationFocus,
+  transformerNotationHighlight,
+  type TransformerCompactLineOption
+} from 'shikiji-transformers'
 
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)
 
@@ -29,7 +29,7 @@ const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)
  * 2. convert line numbers into line options:
  *    [{ line: number, classes: string[] }]
  */
-const attrsToLines = (attrs: string): HtmlRendererOptions['lineOptions'] => {
+const attrsToLines = (attrs: string): TransformerCompactLineOption[] => {
   attrs = attrs.replace(/^(?:\[.*?\])?.*?([\d,-]+).*/, '$1').trim()
   const result: number[] = []
   if (!attrs) {
@@ -53,37 +53,38 @@ const attrsToLines = (attrs: string): HtmlRendererOptions['lineOptions'] => {
   }))
 }
 
-const errorLevelProcessor = defineProcessor({
-  name: 'error-level',
-  handler: createRangeProcessor({
-    error: ['highlighted', 'error'],
-    warning: ['highlighted', 'warning']
-  })
-})
-
 export async function highlight(
   theme: ThemeOptions,
-  languages: ILanguageRegistration[] = [],
+  languages?: LanguageInput[],
   defaultLang: string = '',
-  logger: Pick<Logger, 'warn'> = console
+  logger: Pick<Logger, 'warn'> = console,
+  userTransformers: ShikijiTransformer[] = [],
+  languageAlias: Record<string, string> = {}
 ): Promise<(str: string, lang: string, attrs: string) => string> {
-  const hasSingleTheme = typeof theme === 'string' || 'name' in theme
-  const getThemeName = (themeValue: IThemeRegistration) =>
-    typeof themeValue === 'string' ? themeValue : themeValue.name
-
-  const processors: Processor[] = [
-    createFocusProcessor(),
-    createHighlightProcessor({ hasHighlightClass: 'highlighted' }),
-    createDiffProcessor(),
-    errorLevelProcessor
-  ]
-
   const highlighter = await getHighlighter({
-    themes: hasSingleTheme ? [theme] : [theme.dark, theme.light],
-    langs: [...BUNDLED_LANGUAGES, ...languages],
-    processors
+    themes:
+      typeof theme === 'string' || 'name' in theme
+        ? [theme]
+        : [theme.light, theme.dark],
+    langs: languages?.length ? languages : Object.keys(bundledLanguages),
+    langAlias: languageAlias
   })
 
+  const transformers: ShikijiTransformer[] = [
+    transformerNotationDiff(),
+    transformerNotationFocus({
+      classActiveLine: 'has-focus',
+      classActivePre: 'has-focused-lines'
+    }),
+    transformerNotationHighlight(),
+    transformerNotationErrorLevel(),
+    {
+      pre(node) {
+        addClassToHast(node, 'vp-code')
+      }
+    }
+  ]
+
   const styleRE = /<pre[^>]*(style=".*?")/
   const preRE = /^<pre(.*?)>/
   const vueRE = /-vue$/
@@ -102,7 +103,7 @@ export async function highlight(
 
     if (lang) {
       const langLoaded = highlighter.getLoadedLanguages().includes(lang as any)
-      if (!langLoaded && !['ansi', 'plaintext', 'txt', 'text'].includes(lang)) {
+      if (!langLoaded && !isPlainLang(lang) && !isSpecialLang(lang)) {
         logger.warn(
           c.yellow(
             `\nThe language '${lang}' is not loaded, falling back to '${
@@ -155,24 +156,21 @@ export async function highlight(
 
     str = removeMustache(str).trimEnd()
 
-    const codeToHtml = (theme: IThemeRegistration) => {
-      const res =
-        lang === 'ansi'
-          ? highlighter.ansiToHtml(str, {
-              lineOptions,
-              theme: getThemeName(theme)
-            })
-          : highlighter.codeToHtml(str, {
-              lang,
-              lineOptions,
-              theme: getThemeName(theme)
-            })
-      return fillEmptyHighlightedLine(cleanup(restoreMustache(res)))
-    }
+    const highlighted = highlighter.codeToHtml(str, {
+      lang,
+      transformers: [
+        ...transformers,
+        transformerCompactLineOptions(lineOptions),
+        ...userTransformers
+      ],
+      ...(typeof theme === 'string' || 'name' in theme
+        ? { theme }
+        : {
+            themes: theme,
+            defaultColor: false
+          })
+    })
 
-    if (hasSingleTheme) return codeToHtml(theme)
-    const dark = addClass(codeToHtml(theme.dark), 'vp-code-dark', 'pre')
-    const light = addClass(codeToHtml(theme.light), 'vp-code-light', 'pre')
-    return dark + light
+    return fillEmptyHighlightedLine(cleanup(restoreMustache(highlighted)))
   }
 }

src/node/markdown/plugins/highlight.ts

@@ -4,11 +4,11 @@ This page demonstrates some of the built-in markdown extensions provided by Vite
 
 ## Syntax Highlighting
 
-VitePress provides Syntax Highlighting powered by [Shiki](https://github.com/shikijs/shiki), with additional features like line-highlighting:
+VitePress provides Syntax Highlighting powered by [Shikiji](https://github.com/antfu/shikiji), with additional features like line-highlighting:
 
 **Input**
 
-````
+````md
 ```js{4}
 export default {
   data () {