feat(core): expose flatTokenVariants utility, refactor utils folder

Author: antfu

packages/core/src/highlight/code-to-tokens.ts

@@ -1,7 +1,7 @@
-import type { CodeToTokensOptions, GrammarState, ShikiInternal, ThemedToken, ThemedTokenWithVariants, TokensResult } from '@shikijs/types'
+import type { CodeToTokensOptions, GrammarState, ShikiInternal, ThemedToken, TokensResult } from '@shikijs/types'
 import { ShikiError } from '@shikijs/types'
 import { getLastGrammarStateFromMap, setLastGrammarStateToMap } from '../textmate/grammar-state'
-import { applyColorReplacements, getTokenStyleObject, resolveColorReplacements } from '../utils'
+import { applyColorReplacements, flatTokenVariants, resolveColorReplacements } from '../utils'
 import { codeToTokensBase } from './code-to-tokens-base'
 import { codeToTokensWithThemes } from './code-to-tokens-themes'
 
@@ -51,7 +51,7 @@ export function codeToTokens(
     const themeRegs = themes.map(t => internal.getTheme(t.theme))
     const themesOrder = themes.map(t => t.color)
     tokens = themeTokens
-      .map(line => line.map(token => mergeToken(token, themesOrder, cssVariablePrefix, defaultColor)))
+      .map(line => line.map(token => flatTokenVariants(token, themesOrder, cssVariablePrefix, defaultColor)))
 
     if (grammarState)
       setLastGrammarStateToMap(tokens, grammarState)
@@ -95,40 +95,3 @@ export function codeToTokens(
     grammarState,
   }
 }
-
-function mergeToken(
-  merged: ThemedTokenWithVariants,
-  variantsOrder: string[],
-  cssVariablePrefix: string,
-  defaultColor: string | boolean,
-): ThemedToken {
-  const token: ThemedToken = {
-    content: merged.content,
-    explanation: merged.explanation,
-    offset: merged.offset,
-  }
-
-  const styles = variantsOrder.map(t => getTokenStyleObject(merged.variants[t]))
-
-  // Get all style keys, for themes that missing some style, we put `inherit` to override as needed
-  const styleKeys = new Set(styles.flatMap(t => Object.keys(t)))
-  const mergedStyles: Record<string, string> = {}
-
-  styles.forEach((cur, idx) => {
-    for (const key of styleKeys) {
-      const value = cur[key] || 'inherit'
-
-      if (idx === 0 && defaultColor) {
-        mergedStyles[key] = value
-      }
-      else {
-        const keyName = key === 'color' ? '' : key === 'background-color' ? '-bg' : `-${key}`
-        const varKey = cssVariablePrefix + variantsOrder[idx] + (key === 'color' ? '' : keyName)
-        mergedStyles[varKey] = value
-      }
-    }
-  })
-
-  token.htmlStyle = mergedStyles
-  return token
-}

packages/core/src/textmate/grammar-state.ts

@@ -1,8 +1,8 @@
 import type { GrammarState as GrammarStateInterface, GrammarStateMapKey } from '@shikijs/types'
 import type { StateStack, StateStackImpl } from '@shikijs/vscode-textmate'
 
+import { ShikiError } from '@shikijs/types'
 import { INITIAL } from '@shikijs/vscode-textmate'
-import { ShikiError } from '../../../types/src/error'
 import { toArray } from '../utils'
 import { warnDeprecated } from '../warn'
 

packages/core/src/transformer-decorations.ts

@@ -9,7 +9,7 @@ import type {
   ShikiTransformerContextSource,
 } from '@shikijs/types'
 import type { Element, ElementContent } from 'hast'
-import { ShikiError } from '../../types/src/error'
+import { ShikiError } from '@shikijs/types'
 import { addClassToHast, createPositionConverter, splitTokens } from './utils'
 
 interface TransformerDecorationsInternalContext {

packages/core/src/utils.ts

@@ -0,0 +1,24 @@
+import type { ThemeRegistrationAny, TokenizeWithThemeOptions } from '@shikijs/types'
+
+export function resolveColorReplacements(
+  theme: ThemeRegistrationAny | string,
+  options?: TokenizeWithThemeOptions,
+): Record<string, string | undefined> {
+  const replacements = typeof theme === 'string' ? {} : { ...theme.colorReplacements }
+  const themeName = typeof theme === 'string' ? theme : theme.name
+  for (const [key, value] of Object.entries(options?.colorReplacements || {})) {
+    if (typeof value === 'string')
+      replacements[key] = value
+    else if (key === themeName)
+      Object.assign(replacements, value)
+  }
+  return replacements
+}
+
+export function applyColorReplacements(color: string, replacements?: Record<string, string | undefined>): string
+export function applyColorReplacements(color?: string | undefined, replacements?: Record<string, string | undefined>): string | undefined
+export function applyColorReplacements(color?: string, replacements?: Record<string, string | undefined>): string | undefined {
+  if (!color)
+    return color
+  return replacements?.[color?.toLowerCase()] || color
+}

packages/core/src/utils/colors.ts

@@ -0,0 +1,55 @@
+import type {
+  MaybeArray,
+  MaybeGetter,
+  PlainTextLanguage,
+  SpecialLanguage,
+  SpecialTheme,
+  ThemeInput,
+} from '@shikijs/types'
+
+export function toArray<T>(x: MaybeArray<T>): T[] {
+  return Array.isArray(x) ? x : [x]
+}
+
+/**
+ * Normalize a getter to a promise.
+ */
+export async function normalizeGetter<T>(p: MaybeGetter<T>): Promise<T> {
+  return Promise.resolve(typeof p === 'function' ? (p as any)() : p).then(r => r.default || r)
+}
+
+/**
+ * Check if the language is plaintext that is ignored by Shiki.
+ *
+ * Hard-coded plain text languages: `plaintext`, `txt`, `text`, `plain`.
+ */
+export function isPlainLang(lang: string | null | undefined): lang is PlainTextLanguage {
+  return !lang || ['plaintext', 'txt', 'text', 'plain'].includes(lang)
+}
+
+/**
+ * Check if the language is specially handled or bypassed by Shiki.
+ *
+ * Hard-coded languages: `ansi` and plaintexts like `plaintext`, `txt`, `text`, `plain`.
+ */
+export function isSpecialLang(lang: any): lang is SpecialLanguage {
+  return lang === 'ansi' || isPlainLang(lang)
+}
+
+/**
+ * Check if the theme is specially handled or bypassed by Shiki.
+ *
+ * Hard-coded themes: `none`.
+ */
+export function isNoneTheme(theme: string | ThemeInput | null | undefined): theme is 'none' {
+  return theme === 'none'
+}
+
+/**
+ * Check if the theme is specially handled or bypassed by Shiki.
+ *
+ * Hard-coded themes: `none`.
+ */
+export function isSpecialTheme(theme: string | ThemeInput | null | undefined): theme is SpecialTheme {
+  return isNoneTheme(theme)
+}

packages/core/src/utils/general.ts

@@ -0,0 +1,24 @@
+import type { Element } from 'hast'
+
+/**
+ * Utility to append class to a hast node
+ *
+ * If the `property.class` is a string, it will be splitted by space and converted to an array.
+ */
+export function addClassToHast(node: Element, className: string | string[]): Element {
+  if (!className)
+    return node
+  node.properties ||= {}
+  node.properties.class ||= []
+  if (typeof node.properties.class === 'string')
+    node.properties.class = node.properties.class.split(/\s+/g)
+  if (!Array.isArray(node.properties.class))
+    node.properties.class = []
+
+  const targets = Array.isArray(className) ? className : className.split(/\s+/g)
+  for (const c of targets) {
+    if (c && !node.properties.class.includes(c))
+      node.properties.class.push(c)
+  }
+  return node
+}

packages/core/src/utils/hast.ts

@@ -0,0 +1,5 @@
+export * from './colors'
+export * from './general'
+export * from './hast'
+export * from './strings'
+export * from './tokens'

packages/core/src/utils/index.ts

@@ -0,0 +1,99 @@
+import type { HighlighterGeneric, Position } from '@shikijs/types'
+
+/**
+ * Split a string into lines, each line preserves the line ending.
+ */
+export function splitLines(code: string, preserveEnding = false): [string, number][] {
+  const parts = code.split(/(\r?\n)/g)
+  let index = 0
+  const lines: [string, number][] = []
+  for (let i = 0; i < parts.length; i += 2) {
+    const line = preserveEnding
+      ? parts[i] + (parts[i + 1] || '')
+      : parts[i]
+    lines.push([line, index])
+    index += parts[i].length
+    index += parts[i + 1]?.length || 0
+  }
+  return lines
+}
+
+/**
+ * Creates a converter between index and position in a code block.
+ *
+ * Overflow/underflow are unchecked.
+ */
+export function createPositionConverter(code: string): {
+  lines: string[]
+  indexToPos: (index: number) => Position
+  posToIndex: (line: number, character: number) => number
+} {
+  const lines = splitLines(code, true).map(([line]) => line)
+
+  function indexToPos(index: number): Position {
+    if (index === code.length) {
+      return {
+        line: lines.length - 1,
+        character: lines[lines.length - 1].length,
+      }
+    }
+
+    let character = index
+    let line = 0
+    for (const lineText of lines) {
+      if (character < lineText.length)
+        break
+      character -= lineText.length
+      line++
+    }
+    return { line, character }
+  }
+
+  function posToIndex(line: number, character: number): number {
+    let index = 0
+    for (let i = 0; i < line; i++)
+      index += lines[i].length
+
+    index += character
+    return index
+  }
+
+  return {
+    lines,
+    indexToPos,
+    posToIndex,
+  }
+}
+
+/**
+ * Guess embedded languages from given code and highlighter.
+ *
+ * When highlighter is provided, only bundled languages will be included.
+ */
+export function guessEmbeddedLanguages(
+  code: string,
+  _lang: string | undefined,
+  highlighter?: HighlighterGeneric<any, any>,
+): string[] {
+  const langs = new Set<string>()
+  // For HTML code blocks like Vue SFC
+  for (const match of code.matchAll(/lang=["']([\w-]+)["']/g)) {
+    langs.add(match[1])
+  }
+  // For markdown code blocks
+  for (const match of code.matchAll(/(?:```|~~~)([\w-]+)/g)) {
+    langs.add(match[1])
+  }
+  // For latex
+  for (const match of code.matchAll(/\\begin\{([\w-]+)\}/g)) {
+    langs.add(match[1])
+  }
+
+  if (!highlighter)
+    return Array.from(langs)
+
+  // Only include known languages
+  const bundle = highlighter.getBundledLanguages()
+  return Array.from(langs)
+    .filter(l => l && bundle[l])
+}

packages/core/src/utils/strings.ts

@@ -0,0 +1,129 @@
+import type { ThemedToken, ThemedTokenWithVariants, TokenStyles } from '@shikijs/types'
+import { FontStyle } from '@shikijs/vscode-textmate'
+
+/**
+ * Split a token into multiple tokens by given offsets.
+ *
+ * The offsets are relative to the token, and should be sorted.
+ */
+export function splitToken<
+  T extends Pick<ThemedToken, 'content' | 'offset'>,
+>(
+  token: T,
+  offsets: number[],
+): T[] {
+  let lastOffset = 0
+  const tokens: T[] = []
+
+  for (const offset of offsets) {
+    if (offset > lastOffset) {
+      tokens.push({
+        ...token,
+        content: token.content.slice(lastOffset, offset),
+        offset: token.offset + lastOffset,
+      })
+    }
+    lastOffset = offset
+  }
+
+  if (lastOffset < token.content.length) {
+    tokens.push({
+      ...token,
+      content: token.content.slice(lastOffset),
+      offset: token.offset + lastOffset,
+    })
+  }
+
+  return tokens
+}
+
+/**
+ * Split 2D tokens array by given breakpoints.
+ */
+export function splitTokens<
+  T extends Pick<ThemedToken, 'content' | 'offset'>,
+>(
+  tokens: T[][],
+  breakpoints: number[] | Set<number>,
+): T[][] {
+  const sorted = Array
+    .from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints))
+    .sort((a, b) => a - b)
+
+  if (!sorted.length)
+    return tokens
+
+  return tokens.map((line) => {
+    return line.flatMap((token) => {
+      const breakpointsInToken = sorted
+        .filter(i => token.offset < i && i < token.offset + token.content.length)
+        .map(i => i - token.offset)
+        .sort((a, b) => a - b)
+
+      if (!breakpointsInToken.length)
+        return token
+
+      return splitToken(token, breakpointsInToken)
+    })
+  })
+}
+
+export function flatTokenVariants(
+  merged: ThemedTokenWithVariants,
+  variantsOrder: string[],
+  cssVariablePrefix: string,
+  defaultColor: string | boolean,
+): ThemedToken {
+  const token: ThemedToken = {
+    content: merged.content,
+    explanation: merged.explanation,
+    offset: merged.offset,
+  }
+
+  const styles = variantsOrder.map(t => getTokenStyleObject(merged.variants[t]))
+
+  // Get all style keys, for themes that missing some style, we put `inherit` to override as needed
+  const styleKeys = new Set(styles.flatMap(t => Object.keys(t)))
+  const mergedStyles: Record<string, string> = {}
+
+  styles.forEach((cur, idx) => {
+    for (const key of styleKeys) {
+      const value = cur[key] || 'inherit'
+
+      if (idx === 0 && defaultColor) {
+        mergedStyles[key] = value
+      }
+      else {
+        const keyName = key === 'color' ? '' : key === 'background-color' ? '-bg' : `-${key}`
+        const varKey = cssVariablePrefix + variantsOrder[idx] + (key === 'color' ? '' : keyName)
+        mergedStyles[varKey] = value
+      }
+    }
+  })
+
+  token.htmlStyle = mergedStyles
+  return token
+}
+
+export function getTokenStyleObject(token: TokenStyles): Record<string, string> {
+  const styles: Record<string, string> = {}
+  if (token.color)
+    styles.color = token.color
+  if (token.bgColor)
+    styles['background-color'] = token.bgColor
+  if (token.fontStyle) {
+    if (token.fontStyle & FontStyle.Italic)
+      styles['font-style'] = 'italic'
+    if (token.fontStyle & FontStyle.Bold)
+      styles['font-weight'] = 'bold'
+    if (token.fontStyle & FontStyle.Underline)
+      styles['text-decoration'] = 'underline'
+  }
+  return styles
+}
+
+export function stringifyTokenStyle(token: string | Record<string, string>): string {
+  if (typeof token === 'string')
+    return token
+  return Object.entries(token).map(([key, value]) => `${key}:${value}`).join(';')
+}