feat: introduce experimental JavaScript RegExp Engine (#761)

Author: antfu

.gitignore

@@ -15,3 +15,4 @@ packages/shiki/src/assets/themes
 packages/shiki/src/assets/*.json
 cache
 .eslintcache
+report-engine-js-compat.json

bench/engines.bench.ts

@@ -0,0 +1,39 @@
+import fs from 'node:fs/promises'
+import { bench, describe } from 'vitest'
+import type { BundledLanguage } from 'shiki'
+import { createHighlighter, createJavaScriptRegexEngine, createWasmOnigEngine } from 'shiki'
+import type { ReportItem } from '../scripts/report-engine-js-compat'
+
+describe('engines', async () => {
+  const js = createJavaScriptRegexEngine()
+  const wasm = await createWasmOnigEngine(() => import('shiki/wasm'))
+
+  // Run `npx jiti scripts/report-engine-js-compat.ts` to generate the report first
+  const report = await fs.readFile('../scripts/report-engine-js-compat.json', 'utf-8').then(JSON.parse) as ReportItem[]
+  const langs = report.filter(i => i.highlightMatch === true).map(i => i.lang) as BundledLanguage[]
+  const samples = await Promise.all(langs.map(lang => fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')))
+
+  const shikiJs = await createHighlighter({
+    langs,
+    themes: ['vitesse-dark'],
+    engine: js,
+  })
+
+  const shikiWasm = await createHighlighter({
+    langs,
+    themes: ['vitesse-dark'],
+    engine: wasm,
+  })
+
+  bench('js', () => {
+    for (const lang of langs) {
+      shikiJs.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
+    }
+  }, { warmupIterations: 10, iterations: 30 })
+
+  bench('wasm', () => {
+    for (const lang of langs) {
+      shikiWasm.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
+    }
+  }, { warmupIterations: 10, iterations: 30 })
+})

package.json

@@ -2,7 +2,7 @@
   "type": "module",
   "version": "1.14.1",
   "private": true,
-  "packageManager": "pnpm@9.8.0",
+  "packageManager": "pnpm@9.9.0",
   "scripts": {
     "lint": "eslint . --cache",
     "release": "bumpp && pnpm -r publish",
@@ -46,6 +46,7 @@
     "mdast-util-gfm": "catalog:",
     "mdast-util-to-hast": "catalog:",
     "ofetch": "catalog:",
+    "picocolors": "catalog:",
     "pnpm": "catalog:",
     "prettier": "catalog:",
     "rimraf": "catalog:",

packages/core/package.json

@@ -70,6 +70,7 @@
   },
   "devDependencies": {
     "hast-util-to-html": "catalog:",
+    "oniguruma-to-js": "catalog:",
     "vscode-oniguruma": "catalog:"
   }
 }

packages/core/src/engines/javascript.ts

@@ -0,0 +1,133 @@
+import { onigurumaToRegexp } from 'oniguruma-to-js'
+import type { PatternScanner, RegexEngine, RegexEngineString } from '../textmate'
+import type { JavaScriptRegexEngineOptions } from '../types/engines'
+
+const MAX = 4294967295
+
+export class JavaScriptScanner implements PatternScanner {
+  regexps: (RegExp | null)[]
+
+  constructor(
+    public patterns: string[],
+    public cache: Map<string, RegExp | Error>,
+    public forgiving: boolean,
+  ) {
+    this.regexps = patterns.map((p) => {
+      const cached = cache?.get(p)
+      if (cached) {
+        if (cached instanceof RegExp) {
+          return cached
+        }
+        if (forgiving)
+          return null
+        throw cached
+      }
+      try {
+        const regex = onigurumaToRegexp(
+          p
+            // YAML specific handling; TODO: move to tm-grammars
+            .replaceAll('[^\\s[-?:,\\[\\]{}#&*!|>\'"%@`]]', '[^\\s\\-?:,\\[\\]{}#&*!|>\'"%@`]'),
+          { flags: 'dg' },
+        )
+        cache?.set(p, regex)
+        return regex
+      }
+      catch (e) {
+        cache?.set(p, e as Error)
+        if (forgiving)
+          return null
+        // console.error({ ...e })
+        throw e
+      }
+    })
+  }
+
+  findNextMatchSync(string: string | RegexEngineString, startPosition: number) {
+    const str = typeof string === 'string'
+      ? string
+      : string.content
+    const pending: [index: number, match: RegExpExecArray][] = []
+
+    function toResult(index: number, match: RegExpExecArray) {
+      return {
+        index,
+        captureIndices: match.indices!.map((indice) => {
+          if (indice == null) {
+            return {
+              end: MAX,
+              start: MAX,
+              length: 0,
+            }
+          }
+          return {
+            start: indice[0],
+            length: indice[1] - indice[0],
+            end: indice[1],
+          }
+        }),
+      }
+    }
+
+    for (let i = 0; i < this.regexps.length; i++) {
+      const regexp = this.regexps[i]
+      if (!regexp)
+        continue
+      try {
+        regexp.lastIndex = startPosition
+        const match = regexp.exec(str)
+        if (!match)
+          continue
+        // If the match is at the start position, return it immediately
+        if (match.index === startPosition) {
+          return toResult(i, match)
+        }
+        // Otherwise, store it for later
+        pending.push([i, match])
+      }
+      catch (e) {
+        if (this.forgiving)
+          continue
+        throw e
+      }
+    }
+
+    // Find the closest match to the start position
+    if (pending.length) {
+      const minIndex = Math.min(...pending.map(m => m[1].index))
+      for (const [i, match] of pending) {
+        if (match.index === minIndex) {
+          return toResult(i, match)
+        }
+      }
+    }
+
+    return null
+  }
+}
+
+/**
+ * Use the modern JavaScript RegExp engine to implement the OnigScanner.
+ *
+ * As Oniguruma regex is more powerful than JavaScript regex, some patterns may not be supported.
+ * Errors will be thrown when parsing TextMate grammars with unsupported patterns.
+ * Set `forgiving` to `true` to ignore these errors and skip the unsupported patterns.
+ *
+ * @experimental
+ */
+export function createJavaScriptRegexEngine(options: JavaScriptRegexEngineOptions = {}): RegexEngine {
+  const {
+    forgiving = false,
+    cache = new Map(),
+  } = options
+
+  return {
+    createScanner(patterns: string[]) {
+      return new JavaScriptScanner(patterns, cache, forgiving)
+    },
+    createString(s: string) {
+      return {
+        content: s,
+      }
+    },
+  }
+}

packages/core/src/oniguruma/LICENSE packages/core/src/engines/oniguruma/LICENSE

@@ -2,10 +2,15 @@
  * Copyright (C) Microsoft Corporation. All rights reserved.
  *-------------------------------------------------------- */
 
-import { ShikiError } from '../error'
-import type { IOnigBinding, IOnigCaptureIndex, IOnigMatch, OnigScanner as IOnigScanner, OnigString as IOnigString, Pointer } from './types'
+import { ShikiError } from '../../error'
+import type { LoadWasmOptions, WebAssemblyInstance, WebAssemblyInstantiator } from '../../types'
+import type { IOnigCaptureIndex, IOnigMatch, OnigScanner as IOnigScanner, OnigString as IOnigString } from '../../../vendor/vscode-textmate/src/main'
 import createOnigasm from './onig'
 
+export type Instantiator = (importObject: Record<string, Record<string, WebAssembly.ImportValue>>) => Promise<WebAssembly.Exports>
+
+export type Pointer = number
+
 export const enum FindOption {
   None = 0,
   /**
@@ -20,14 +25,25 @@ export const enum FindOption {
    * equivalent of ONIG_OPTION_NOT_BEGIN_POSITION: (start) isn't considered as start position of search (* fail \G)
    */
   NotBeginPosition = 4,
-  /**
-   * used for debugging purposes.
-   */
-  DebugCall = 8,
+}
+
+export interface IOnigBinding {
+  HEAPU8: Uint8Array
+  HEAPU32: Uint32Array
+
+  UTF8ToString: (ptr: Pointer) => string
+
+  omalloc: (count: number) => Pointer
+  ofree: (ptr: Pointer) => void
+  getLastOnigError: () => Pointer
+  createOnigScanner: (strPtrsPtr: Pointer, strLenPtr: Pointer, count: number) => Pointer
+  freeOnigScanner: (ptr: Pointer) => void
+  findNextOnigScannerMatch: (scanner: Pointer, strCacheId: number, strData: Pointer, strLength: number, position: number, options: number) => number
+  // findNextOnigScannerMatchDbg: (scanner: Pointer, strCacheId: number, strData: Pointer, strLength: number, position: number, options: number) => number
 }
 
 let onigBinding: IOnigBinding | null = null
-let defaultDebugCall = false
+// let defaultDebugCall = false
 
 function throwLastOnigError(onigBinding: IOnigBinding): void {
   throw new ShikiError(onigBinding.UTF8ToString(onigBinding.getLastOnigError()))
@@ -294,34 +310,33 @@ export class OnigScanner implements IOnigScanner {
   public findNextMatchSync(string: string | OnigString, startPosition: number, debugCall: boolean): IOnigMatch | null
   public findNextMatchSync(string: string | OnigString, startPosition: number): IOnigMatch | null
   public findNextMatchSync(string: string | OnigString, startPosition: number, arg?: number | boolean): IOnigMatch | null {
-    let debugCall = defaultDebugCall
+    // let debugCall = defaultDebugCall
     let options = FindOption.None
     if (typeof arg === 'number') {
-      if (arg & FindOption.DebugCall)
-        debugCall = true
-
+      // if (arg & FindOption.DebugCall)
+      //   debugCall = true
       options = arg
     }
     else if (typeof arg === 'boolean') {
-      debugCall = arg
+      // debugCall = arg
     }
     if (typeof string === 'string') {
       string = new OnigString(string)
-      const result = this._findNextMatchSync(string, startPosition, debugCall, options)
+      const result = this._findNextMatchSync(string, startPosition, false, options)
       string.dispose()
       return result
     }
-    return this._findNextMatchSync(string, startPosition, debugCall, options)
+    return this._findNextMatchSync(string, startPosition, false, options)
   }
 
   private _findNextMatchSync(string: OnigString, startPosition: number, debugCall: boolean, options: number): IOnigMatch | null {
     const onigBinding = this._onigBinding
-    let resultPtr: Pointer
-    if (debugCall)
-      resultPtr = onigBinding.findNextOnigScannerMatchDbg(this._ptr, string.id, string.ptr, string.utf8Length, string.convertUtf16OffsetToUtf8(startPosition), options)
+    // let resultPtr: Pointer
+    // if (debugCall)
+    //   resultPtr = onigBinding.findNextOnigScannerMatchDbg(this._ptr, string.id, string.ptr, string.utf8Length, string.convertUtf16OffsetToUtf8(startPosition), options)
 
-    else
-      resultPtr = onigBinding.findNextOnigScannerMatch(this._ptr, string.id, string.ptr, string.utf8Length, string.convertUtf16OffsetToUtf8(startPosition), options)
+    // else
+    const resultPtr = onigBinding.findNextOnigScannerMatch(this._ptr, string.id, string.ptr, string.utf8Length, string.convertUtf16OffsetToUtf8(startPosition), options)
 
     if (resultPtr === 0) {
       // no match
@@ -348,17 +363,6 @@ export class OnigScanner implements IOnigScanner {
   }
 }
 
-export interface WebAssemblyInstantiator {
-  (importObject: Record<string, Record<string, WebAssembly.ImportValue>> | undefined): Promise<WebAssemblyInstance>
-}
-
-export type WebAssemblyInstance = WebAssembly.WebAssemblyInstantiatedSource | WebAssembly.Instance | WebAssembly.Instance['exports']
-
-export type OnigurumaLoadOptions =
-  | { instantiator: WebAssemblyInstantiator }
-  | { default: WebAssemblyInstantiator }
-  | { data: ArrayBufferView | ArrayBuffer | Response }
-
 function isInstantiatorOptionsObject(dataOrOptions: any): dataOrOptions is { instantiator: WebAssemblyInstantiator } {
   return (typeof dataOrOptions.instantiator === 'function')
 }
@@ -385,15 +389,6 @@ function isArrayBuffer(data: any): data is ArrayBuffer | ArrayBufferView {
 
 let initPromise: Promise<void>
 
-type Awaitable<T> = T | Promise<T>
-
-export type LoadWasmOptionsPlain =
-  | OnigurumaLoadOptions
-  | WebAssemblyInstantiator
-  | ArrayBufferView | ArrayBuffer | Response
-
-export type LoadWasmOptions = Awaitable<LoadWasmOptionsPlain> | (() => Awaitable<LoadWasmOptionsPlain>)
-
 export function loadWasm(options: LoadWasmOptions): Promise<void> {
   if (initPromise)
     return initPromise
@@ -461,14 +456,14 @@ function _makeResponseNonStreamingLoader(data: Response): WebAssemblyInstantiato
   }
 }
 
-export function createOnigString(str: string) {
-  return new OnigString(str)
-}
+// export function createOnigString(str: string) {
+//   return new OnigString(str)
+// }
 
-export function createOnigScanner(patterns: string[]) {
-  return new OnigScanner(patterns)
-}
+// export function createOnigScanner(patterns: string[]) {
+//   return new OnigScanner(patterns)
+// }
 
-export function setDefaultDebugCall(_defaultDebugCall: boolean): void {
-  defaultDebugCall = _defaultDebugCall
-}
+// export function setDefaultDebugCall(_defaultDebugCall: boolean): void {
+//   defaultDebugCall = _defaultDebugCall
+// }

packages/core/src/oniguruma/index.ts packages/core/src/engines/oniguruma/index.ts

@@ -1,4 +1,14 @@
-import type { IOnigBinding, Instantiator } from './types'
+import type { IOnigBinding, Instantiator } from '.'
+
+function getHeapMax() {
+  return 2147483648
+}
+
+function _emscripten_get_now() {
+  return typeof performance !== 'undefined' ? performance.now() : Date.now()
+}
+
+const alignUp = (x: number, multiple: number) => x + ((multiple - (x % multiple)) % multiple)
 
 export default async function main(init: Instantiator): Promise<IOnigBinding> {
   let wasmMemory: any
@@ -12,15 +22,10 @@ export default async function main(init: Instantiator): Promise<IOnigBinding> {
     binding.HEAPU32 = new Uint32Array(buf)
   }
 
-  function _emscripten_get_now() {
-    return typeof performance !== 'undefined' ? performance.now() : Date.now()
-  }
   function _emscripten_memcpy_big(dest: number, src: number, num: number) {
     binding.HEAPU8.copyWithin(dest, src, src + num)
   }
-  function getHeapMax() {
-    return 2147483648
-  }
+
   function emscripten_realloc_buffer(size: number) {
     try {
       wasmMemory.grow((size - buffer.byteLength + 65535) >>> 16)
@@ -36,7 +41,6 @@ export default async function main(init: Instantiator): Promise<IOnigBinding> {
     if (requestedSize > maxHeapSize)
       return false
 
-    const alignUp = (x: number, multiple: number) => x + ((multiple - (x % multiple)) % multiple)
     for (let cutDown = 1; cutDown <= 4; cutDown *= 2) {
       let overGrownHeapSize = oldSize * (1 + 0.2 / cutDown)
       overGrownHeapSize = Math.min(overGrownHeapSize, requestedSize + 100663296)

packages/core/src/oniguruma/onig.ts packages/core/src/engines/oniguruma/onig.ts

@@ -0,0 +1,19 @@
+import type { RegexEngine } from '../textmate'
+import type { LoadWasmOptions } from '../types'
+import { OnigScanner, OnigString, loadWasm } from './oniguruma'
+
+export { loadWasm }
+
+export async function createWasmOnigEngine(options?: LoadWasmOptions | null): Promise<RegexEngine> {
+  if (options)
+    await loadWasm(options)
+
+  return {
+    createScanner(patterns) {
+      return new OnigScanner(patterns)
+    },
+    createString(s) {
+      return new OnigString(s)
+    },
+  }
+}

packages/core/src/engines/wasm.ts

@@ -3,7 +3,9 @@ export * from './bundle-factory'
 export * from './utils'
 export * from './types'
 
-export { loadWasm } from './oniguruma'
+export { createWasmOnigEngine, loadWasm } from './engines/wasm'
+export { createJavaScriptRegexEngine } from './engines/javascript'
+
 export { createShikiInternal, getShikiInternal, setDefaultWasmLoader } from './internal'
 export { codeToTokensBase, tokenizeWithTheme } from './code-to-tokens-base'
 export { codeToTokens } from './code-to-tokens'

packages/core/src/index.ts

@@ -1,11 +1,10 @@
-import type { HighlighterCoreOptions, LanguageInput, LanguageRegistration, MaybeGetter, ShikiInternal, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved } from './types'
-import type { LoadWasmOptions } from './oniguruma'
-import { createOnigScanner, createOnigString, loadWasm } from './oniguruma'
+import type { HighlighterCoreOptions, LanguageInput, LanguageRegistration, LoadWasmOptions, MaybeGetter, ShikiInternal, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved } from './types'
 import { Registry } from './registry'
 import { Resolver } from './resolver'
 import { normalizeTheme } from './normalize'
 import { isSpecialLang, isSpecialTheme } from './utils'
 import { ShikiError } from './error'
+import { createWasmOnigEngine } from './engines/wasm'
 
 let _defaultWasmLoader: LoadWasmOptions | undefined
 /**
@@ -40,26 +39,16 @@ export async function createShikiInternal(options: HighlighterCoreOptions = {}):
     )).flat()))
   }
 
-  const wasmLoader = options.loadWasm || _defaultWasmLoader
-
   const [
     themes,
     langs,
   ] = await Promise.all([
     Promise.all((options.themes || []).map(normalizeGetter)).then(r => r.map(normalizeTheme)),
     resolveLangs(options.langs || []),
-    wasmLoader ? loadWasm(wasmLoader) : undefined,
   ] as const)
 
   const resolver = new Resolver(
-    Promise.resolve({
-      createOnigScanner(patterns) {
-        return createOnigScanner(patterns)
-      },
-      createOnigString(s) {
-        return createOnigString(s)
-      },
-    }),
+    Promise.resolve(options.engine || createWasmOnigEngine(options.loadWasm || _defaultWasmLoader)),
     langs,
   )
 

packages/core/src/internal.ts

@@ -1,4 +1,5 @@
-import type { IOnigLib, RegistryOptions } from './textmate'
+import type { IOnigLib } from '../vendor/vscode-textmate/src/main'
+import type { RegexEngine, RegistryOptions } from './textmate'
 import type { LanguageRegistration } from './types'
 
 export class Resolver implements RegistryOptions {
@@ -8,8 +9,12 @@ export class Resolver implements RegistryOptions {
 
   private readonly _onigLibPromise: Promise<IOnigLib>
 
-  constructor(onigLibPromise: Promise<IOnigLib>, langs: LanguageRegistration[]) {
+  constructor(onigLibPromise: Promise<RegexEngine>, langs: LanguageRegistration[]) {
     this._onigLibPromise = onigLibPromise
+      .then(engine => ({
+        createOnigScanner: patterns => engine.createScanner(patterns),
+        createOnigString: s => engine.createString(s),
+      }))
     langs.forEach(i => this.addLanguage(i))
   }
 

packages/core/src/oniguruma/types.ts

@@ -1,4 +1,6 @@
 // We re-bundled vscode-textmate from source to have ESM support.
+import type { OnigScanner, OnigString } from '../vendor/vscode-textmate/src/main'
+
 // This file re-exports some runtime values we need.
 export { Registry, INITIAL, StateStack } from '../vendor/vscode-textmate/src/main'
 export { Theme } from '../vendor/vscode-textmate/src/theme'
@@ -7,8 +9,19 @@ export type {
   IRawGrammar,
   IGrammar,
   IGrammarConfiguration,
-  IOnigLib,
   RegistryOptions,
 } from '../vendor/vscode-textmate/src/main'
 export type { IRawThemeSetting } from '../vendor/vscode-textmate/src/theme'
 export * from './stack-element-metadata'
+
+export interface PatternScanner extends OnigScanner {}
+
+export interface RegexEngineString extends OnigString {}
+
+/**
+ * Engine for RegExp matching and scanning.
+ */
+export interface RegexEngine {
+  createScanner: (patterns: string[]) => PatternScanner
+  createString: (s: string) => RegexEngineString
+}

packages/core/src/resolver.ts

@@ -62,29 +62,6 @@ export function transformerDecorations(): ShikiTransformer {
     return map.get(shiki.meta)!
   }
 
-  function verifyIntersections(items: ResolvedDecorationItem[]) {
-    for (let i = 0; i < items.length; i++) {
-      const foo = items[i]
-      if (foo.start.offset > foo.end.offset)
-        throw new ShikiError(`Invalid decoration range: ${JSON.stringify(foo.start)} - ${JSON.stringify(foo.end)}`)
-
-      for (let j = i + 1; j < items.length; j++) {
-        const bar = items[j]
-        const isFooHasBarStart = foo.start.offset < bar.start.offset && bar.start.offset < foo.end.offset
-        const isFooHasBarEnd = foo.start.offset < bar.end.offset && bar.end.offset < foo.end.offset
-        const isBarHasFooStart = bar.start.offset < foo.start.offset && foo.start.offset < bar.end.offset
-        const isBarHasFooEnd = bar.start.offset < foo.end.offset && foo.end.offset < bar.end.offset
-        if (isFooHasBarStart || isFooHasBarEnd || isBarHasFooStart || isBarHasFooEnd) {
-          if (isFooHasBarEnd && isFooHasBarEnd)
-            continue // nested
-          if (isBarHasFooStart && isBarHasFooEnd)
-            continue // nested
-          throw new ShikiError(`Decorations ${JSON.stringify(foo.start)} and ${JSON.stringify(bar.start)} intersect.`)
-        }
-      }
-    }
-  }
-
   return {
     name: 'shiki:decorations',
     tokens(tokens) {
@@ -111,14 +88,6 @@ export function transformerDecorations(): ShikiTransformer {
         let startIndex = -1
         let endIndex = -1
 
-        function stringify(el: ElementContent): string {
-          if (el.type === 'text')
-            return el.value
-          if (el.type === 'element')
-            return el.children.map(stringify).join('')
-          return ''
-        }
-
         if (start === 0)
           startIndex = 0
         if (end === 0)
@@ -206,3 +175,34 @@ export function transformerDecorations(): ShikiTransformer {
     },
   }
 }
+
+function verifyIntersections(items: ResolvedDecorationItem[]) {
+  for (let i = 0; i < items.length; i++) {
+    const foo = items[i]
+    if (foo.start.offset > foo.end.offset)
+      throw new ShikiError(`Invalid decoration range: ${JSON.stringify(foo.start)} - ${JSON.stringify(foo.end)}`)
+
+    for (let j = i + 1; j < items.length; j++) {
+      const bar = items[j]
+      const isFooHasBarStart = foo.start.offset < bar.start.offset && bar.start.offset < foo.end.offset
+      const isFooHasBarEnd = foo.start.offset < bar.end.offset && bar.end.offset < foo.end.offset
+      const isBarHasFooStart = bar.start.offset < foo.start.offset && foo.start.offset < bar.end.offset
+      const isBarHasFooEnd = bar.start.offset < foo.end.offset && foo.end.offset < bar.end.offset
+      if (isFooHasBarStart || isFooHasBarEnd || isBarHasFooStart || isBarHasFooEnd) {
+        if (isFooHasBarEnd && isFooHasBarEnd)
+          continue // nested
+        if (isBarHasFooStart && isBarHasFooEnd)
+          continue // nested
+        throw new ShikiError(`Decorations ${JSON.stringify(foo.start)} and ${JSON.stringify(bar.start)} intersect.`)
+      }
+    }
+  }
+}
+
+function stringify(el: ElementContent): string {
+  if (el.type === 'text')
+    return el.value
+  if (el.type === 'element')
+    return el.children.map(stringify).join('')
+  return ''
+}

packages/core/src/textmate.ts

@@ -0,0 +1,31 @@
+type Awaitable<T> = T | Promise<T>
+
+export interface WebAssemblyInstantiator {
+  (importObject: Record<string, Record<string, WebAssembly.ImportValue>> | undefined): Promise<WebAssemblyInstance>
+}
+
+export type WebAssemblyInstance = WebAssembly.WebAssemblyInstantiatedSource | WebAssembly.Instance | WebAssembly.Instance['exports']
+
+export type OnigurumaLoadOptions =
+  | { instantiator: WebAssemblyInstantiator }
+  | { default: WebAssemblyInstantiator }
+  | { data: ArrayBufferView | ArrayBuffer | Response }
+
+export type LoadWasmOptionsPlain =
+  | OnigurumaLoadOptions
+  | WebAssemblyInstantiator
+  | ArrayBufferView | ArrayBuffer | Response
+
+export type LoadWasmOptions = Awaitable<LoadWasmOptionsPlain> | (() => Awaitable<LoadWasmOptionsPlain>)
+
+export interface JavaScriptRegexEngineOptions {
+  /**
+   * Whether to allow invalid regex patterns.
+   */
+  forgiving?: boolean
+
+  /**
+   * Cache for regex patterns.
+   */
+  cache?: Map<string, RegExp | Error>
+}

packages/core/src/transformer-decorations.ts

@@ -8,4 +8,8 @@ export * from './transformers'
 export * from './utils'
 export * from './decorations'
 
-export { WebAssemblyInstantiator } from '../oniguruma'
+export type {
+  LoadWasmOptions,
+  WebAssemblyInstantiator,
+  WebAssemblyInstance,
+} from './engines'

packages/core/src/types/engines.ts

@@ -1,4 +1,5 @@
-import type { LoadWasmOptions } from '../oniguruma'
+import type { LoadWasmOptions } from '../types'
+import type { RegexEngine } from '../textmate'
 import type { StringLiteralUnion } from './utils'
 import type { LanguageInput, SpecialLanguage } from './langs'
 import type { SpecialTheme, ThemeInput, ThemeRegistrationAny } from './themes'
@@ -29,9 +30,13 @@ export interface HighlighterCoreOptions {
    * @default true
    */
   warnings?: boolean
+  /**
+   * Custom RegExp engine.
+   */
+  engine?: RegexEngine | Promise<RegexEngine>
 }
 
-export interface BundledHighlighterOptions<L extends string, T extends string> {
+export interface BundledHighlighterOptions<L extends string, T extends string> extends Pick<HighlighterCoreOptions, 'warnings' | 'engine'> {
   /**
    * Theme registation
    *

packages/core/src/types/index.ts

@@ -1,6 +1,6 @@
 // @ts-expect-error this will be compiled to ArrayBuffer
 import binary from 'vscode-oniguruma/release/onig.wasm'
-import type { WebAssemblyInstantiator } from './oniguruma'
+import type { WebAssemblyInstantiator } from './types'
 
 export const wasmBinary = binary as ArrayBuffer
 

packages/core/src/types/options.ts

@@ -48,6 +48,7 @@ it('run with rehype-raw', async () => {
     ],
   })
 
+  // eslint-disable-next-line unicorn/consistent-function-scoping
   const rehypeMetaString = () => (tree: Root) => {
     visit(tree, 'element', (node) => {
       if (node.tagName === 'code' && node.data?.meta) {

packages/core/src/wasm-inlined.ts

@@ -0,0 +1,121 @@
+import { describe, expect, it } from 'vitest'
+import { wasmBinary } from '@shikijs/core/wasm-inlined'
+import type { RegexEngine } from '@shikijs/core/textmate'
+import type { LanguageRegistration, ThemeRegistration } from '../../src/core'
+import { createHighlighterCore, createJavaScriptRegexEngine, loadWasm } from '../../src/core'
+
+import { OnigScanner, OnigString } from '../../../core/src/engines/oniguruma'
+import type { Instance } from './types'
+
+await loadWasm({ instantiator: obj => WebAssembly.instantiate(wasmBinary, obj) })
+
+function createWasmOnigLibWrapper(): RegexEngine & { instances: Instance[] } {
+  const instances: Instance[] = []
+
+  return {
+    instances,
+    createScanner(patterns) {
+      const scanner = new OnigScanner(patterns)
+      const instance: Instance = {
+        constractor: [patterns],
+        executions: [],
+      }
+      instances.push(instance)
+      return {
+        findNextMatchSync(string: string | OnigString, startPosition: number) {
+          const result = scanner.findNextMatchSync(string, startPosition)
+          instance.executions.push({ args: [typeof string === 'string' ? string : string.content, startPosition], result })
+          return result
+        },
+      }
+    },
+    createString(s) {
+      return new OnigString(s)
+    },
+  }
+}
+
+export interface Cases {
+  name: string
+  theme: () => Promise<{ default: ThemeRegistration }>
+  lang: () => Promise<{ default: LanguageRegistration[] }>
+  cases: string[]
+}
+
+const cases: Cases[] = [
+  {
+    name: 'json-basic',
+    theme: () => import('../../src/assets/themes/nord'),
+    lang: () => import('../../src/assets/langs/json'),
+    cases: [
+      '{"foo":{"bar":1}}',
+      '[undefined, null, true, false, 0, 1, 1.1, "foo", [], {}]',
+    ],
+  },
+  {
+    name: 'html-basic',
+    theme: () => import('../../src/assets/themes/nord'),
+    lang: () => import('../../src/assets/langs/html'),
+    cases: [
+      '<div class="foo">bar</div>',
+      '<!DOCTYPE html><html><head><title>foo</title></head><body>bar</body></html>',
+    ],
+  },
+  {
+    name: 'ts-basic',
+    theme: () => import('../../src/assets/themes/nord'),
+    lang: () => import('../../src/assets/langs/typescript'),
+    cases: [
+      'const foo: string = "bar"',
+    ],
+  },
+]
+
+describe('cases', async () => {
+  const resolved = await Promise.all(cases.map(async (c) => {
+    const theme = await c.theme().then(r => r.default)
+    const lang = await c.lang().then(r => r.default)
+    return {
+      theme,
+      lang,
+      c,
+    }
+  }))
+
+  for (const c of resolved) {
+    it(c.c.name, async () => {
+      const wasm = createWasmOnigLibWrapper()
+      const native = createJavaScriptRegexEngine()
+
+      const shiki1 = await createHighlighterCore({
+        langs: c.lang,
+        themes: [c.theme],
+        engine: wasm,
+      })
+      const shiki2 = await createHighlighterCore({
+        langs: c.lang,
+        themes: [c.theme],
+        engine: native,
+      })
+
+      const lang = c.lang[0].name
+      const theme = c.theme.name!
+
+      const compare: [any, any][] = []
+
+      for (const code of c.c.cases) {
+        compare.push([
+          shiki1.codeToTokensBase(code, { lang, theme }),
+          shiki2.codeToTokensBase(code, { lang, theme }),
+        ])
+      }
+
+      await expect(JSON.stringify(wasm.instances, null, 2))
+        .toMatchFileSnapshot(`./__records__/${c.c.name}.json`)
+
+      for (const [a, b] of compare) {
+        expect.soft(a).toEqual(b)
+      }
+    })
+  }
+})

packages/rehype/test/core.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest'
+import { createHighlighter, createJavaScriptRegexEngine } from '../../src'
+
+describe('should', () => {
+  it('works', async () => {
+    const shiki = await createHighlighter({
+      themes: ['vitesse-light'],
+      langs: ['javascript'],
+      engine: createJavaScriptRegexEngine(),
+    })
+
+    expect(shiki.codeToHtml('console.log', { lang: 'js', theme: 'vitesse-light' }))
+      .toMatchInlineSnapshot(`"<pre class="shiki vitesse-light" style="background-color:#ffffff;color:#393a34" tabindex="0"><code><span class="line"><span style="color:#B07D48">console</span><span style="color:#999999">.</span><span style="color:#B07D48">log</span></span></code></pre>"`)
+  })
+
+  it('dynamic load theme and lang', async () => {
+    const shiki = await createHighlighter({
+      themes: ['vitesse-light'],
+      langs: ['javascript', 'ts'],
+      engine: createJavaScriptRegexEngine(),
+    })
+
+    await shiki.loadLanguage('css')
+    await shiki.loadTheme('min-dark')
+
+    expect(shiki.getLoadedLanguages())
+      .toMatchInlineSnapshot(`
+        [
+          "javascript",
+          "typescript",
+          "css",
+          "js",
+          "ts",
+        ]
+      `)
+    expect(shiki.getLoadedThemes())
+      .toMatchInlineSnapshot(`
+        [
+          "vitesse-light",
+          "min-dark",
+        ]
+      `)
+
+    expect(shiki.codeToHtml('@media foo { .bar { padding-left: 1px; }}', { lang: 'css', theme: 'min-dark' }))
+      .toMatchInlineSnapshot(`"<pre class="shiki min-dark" style="background-color:#1f1f1f;color:#b392f0" tabindex="0"><code><span class="line"><span style="color:#B392F0">@media foo { .bar { </span><span style="color:#79B8FF">padding-left</span><span style="color:#F97583">:</span><span style="color:#F8F8F8"> 1px</span><span style="color:#B392F0">; }}</span></span></code></pre>"`)
+  })
+})

packages/shiki/test/engine-js/__records__/html-basic.json

@@ -0,0 +1 @@
+<pre class="shiki monokai" style="background-color:#272822;color:#F8F8F2" tabindex="0"><code><span class="line"><span style="color:#66D9EF;font-style:italic">type</span><span> </span><span style="color:#A6E22E;text-decoration:underline">Foo</span><span style="color:#F92672"> =</span><span style="color:#F8F8F2"> { bar</span><span style="color:#F92672">:</span><span style="color:#66D9EF;font-style:italic"> string</span><span style="color:#F8F8F2"> }</span></span></code></pre>

packages/shiki/test/engine-js/__records__/json-basic.json

@@ -0,0 +1,11 @@
+import type { IOnigMatch } from '../../../core/vendor/vscode-textmate/src/main'
+
+export interface Instance {
+  constractor: [string[]]
+  executions: Executions[]
+}
+
+export interface Executions {
+  args: [str: string, start: number]
+  result: IOnigMatch | null
+}

packages/shiki/test/engine-js/__records__/ts-basic.json

@@ -0,0 +1,45 @@
+import { fileURLToPath } from 'node:url'
+import { basename } from 'node:path'
+import { promises as fs } from 'node:fs'
+import { describe, expect, it, onTestFailed } from 'vitest'
+import fg from 'fast-glob'
+import { JavaScriptScanner } from '../../../core/src/engines/javascript'
+import type { Instance } from './types'
+
+const files = await fg('*.json', {
+  cwd: fileURLToPath(new URL('./__records__', import.meta.url)),
+  absolute: true,
+  onlyFiles: true,
+})
+
+const cache = new Map<string, RegExp | Error>()
+
+for (const file of files) {
+  // Some token positions are off in this record
+  const name = basename(file, '.json')
+  if (name === 'ts-basic')
+    continue
+
+  describe(`record: ${name}`, async () => {
+    const instances = JSON.parse(await fs.readFile(file, 'utf-8')) as Instance[]
+    let i = 0
+    for (const instance of instances) {
+      describe(`instances ${i++}`, () => {
+        const scanner = new JavaScriptScanner(instance.constractor[0], cache, false)
+        let j = 0
+        for (const execution of instance.executions) {
+          it(`case ${j++}`, () => {
+            onTestFailed(() => {
+              console.error({
+                patterns: scanner.patterns,
+                regexps: scanner.regexps,
+              })
+            })
+            const result = scanner.findNextMatchSync(...execution.args)
+            expect(result).toEqual(execution.result)
+          })
+        }
+      })
+    }
+  })
+}

packages/shiki/test/engine-js/compare.test.ts

@@ -3,11 +3,11 @@ import type { ThemedToken } from '../src'
 import { codeToHtml, codeToTokensBase, codeToTokensWithThemes } from '../src'
 import { syncThemesTokenization } from '../../core/src/code-to-tokens-themes'
 
-describe('syncThemesTokenization', () => {
-  function stringifyTokens(tokens: ThemedToken[][]) {
-    return tokens.map(line => line.map(token => token.content).join(' ')).join('\n')
-  }
+function stringifyTokens(tokens: ThemedToken[][]) {
+  return tokens.map(line => line.map(token => token.content).join(' ')).join('\n')
+}
 
+describe('syncThemesTokenization', () => {
   it('two themes', async () => {
     const lines1 = await codeToTokensBase('console.log("hello")', { lang: 'js', theme: 'vitesse-dark', includeExplanation: true })
     const lines2 = await codeToTokensBase('console.log("hello")', { lang: 'js', theme: 'min-light', includeExplanation: true })

packages/shiki/test/engine-js/general.test.ts

@@ -11,6 +11,7 @@ it('wasm', async () => {
   const shiki = await createHighlighterCore({
     themes: [nord],
     langs: [js as any],
+    // eslint-disable-next-line unicorn/consistent-function-scoping
     loadWasm: Promise.resolve().then(() => obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance)),
   })
 

packages/shiki/test/engine-js/out/monokai-underline.html

@@ -47,29 +47,6 @@ export function rendererFloatingVue(options: TwoslashFloatingVueRendererOptions
     'theme': floatingVueTheme,
   }
 
-  function compose(parts: { token: Element | Text, popup: Element }): Element[] {
-    return [
-      {
-        type: 'element',
-        tagName: 'span',
-        properties: {},
-        children: [parts.token],
-      },
-      {
-        type: 'element',
-        tagName: 'template',
-        properties: {
-          'v-slot:popper': '{}',
-        },
-        content: {
-          type: 'root',
-          children: [vPre(parts.popup)],
-        },
-        children: [],
-      },
-    ]
-  }
-
   const rich = rendererRich({
     classExtra: classCopyIgnore,
     ...options,
@@ -194,3 +171,26 @@ function renderMarkdownInline(this: ShikiTransformerContextCommon, md: string, c
     return children[0].children
   return children
 }
+
+function compose(parts: { token: Element | Text, popup: Element }): Element[] {
+  return [
+    {
+      type: 'element',
+      tagName: 'span',
+      properties: {},
+      children: [parts.token],
+    },
+    {
+      type: 'element',
+      tagName: 'template',
+      properties: {
+        'v-slot:popper': '{}',
+      },
+      content: {
+        type: 'root',
+        children: [vPre(parts.popup)],
+      },
+      children: [],
+    },
+  ]
+}

packages/shiki/test/engine-js/types.ts

@@ -6,8 +6,8 @@ packages:
   - docs
   - packages/@shikijs/core/vendor/*
 catalog:
-  '@antfu/eslint-config': ^2.27.0
-  '@antfu/ni': ^0.22.4
+  '@antfu/eslint-config': ^3.0.0
+  '@antfu/ni': ^0.23.0
   '@antfu/utils': ^0.7.10
   '@iconify-json/carbon': ^1.1.37
   '@iconify-json/codicon': ^1.1.51
@@ -24,22 +24,22 @@ catalog:
   '@types/hast': ^3.0.4
   '@types/markdown-it': ^14.1.2
   '@types/minimist': ^1.2.5
-  '@types/node': ^22.5.0
-  '@unocss/reset': ^0.62.2
+  '@types/node': ^22.5.1
+  '@unocss/reset': ^0.62.3
   '@vitest/coverage-v8': ^2.0.5
-  '@vueuse/core': ^11.0.1
+  '@vueuse/core': ^11.0.3
   ansi-sequence-parser: ^1.1.1
-  bumpp: ^9.5.1
+  bumpp: ^9.5.2
   chalk: ^5.3.0
-  eslint: ^9.9.0
+  eslint: ^9.9.1
   eslint-plugin-format: ^0.1.2
   esno: ^4.7.0
   fast-glob: ^3.3.2
   floating-vue: ^5.2.2
   fs-extra: ^11.2.0
   fuse.js: ^7.0.0
   hast-util-from-html: ^2.0.2
-  hast-util-to-html: ^9.0.1
+  hast-util-to-html: ^9.0.2
   hast-util-to-string: ^3.0.0
   jsonc-parser: ^3.3.1
   lint-staged: ^15.2.9
@@ -48,40 +48,42 @@ catalog:
   mdast-util-gfm: ^3.0.0
   mdast-util-to-hast: ^13.2.0
   minimist: ^1.2.8
-  monaco-editor-core: ^0.50.0
+  monaco-editor-core: ^0.51.0
   ofetch: ^1.3.4
+  oniguruma-to-js: ^0.2.3
+  picocolors: ^1.0.1
   pinia: ^2.2.2
-  pnpm: ^9.8.0
+  pnpm: ^9.9.0
   prettier: ^3.3.3
   rehype-raw: ^7.0.0
   rehype-stringify: ^10.0.0
   remark-parse: ^11.0.0
   remark-rehype: ^11.1.0
   rimraf: ^6.0.1
-  rollup: ^4.21.0
+  rollup: ^4.21.2
   rollup-plugin-copy: ^3.5.0
   rollup-plugin-dts: ^6.1.1
   rollup-plugin-esbuild: ^6.1.1
   rollup-plugin-typescript2: ^0.36.0
   shiki-legacy: npm:shiki@^0.14.7
   simple-git-hooks: ^2.11.1
-  taze: ^0.16.6
-  tm-grammars: ^1.17.3
-  tm-themes: ^1.8.0
+  taze: ^0.16.7
+  tm-grammars: ^1.17.9
+  tm-themes: ^1.8.1
   twoslash: ^0.2.9
   twoslash-vue: ^0.2.9
   typescript: ^5.5.4
   unbuild: ^2.0.0
   unified: ^11.0.5
   unist-util-visit: ^5.0.0
-  unocss: ^0.62.2
+  unocss: ^0.62.3
   unplugin-vue-components: ^0.27.4
   vite: ^5.4.2
   vite-tsconfig-paths: ^5.0.1
-  vitepress: ^1.3.3
+  vitepress: ^1.3.4
   vitepress-plugin-mermaid: ^2.0.16
   vitest: ^2.0.5
   vscode-oniguruma: ^1.7.0
   vue: ^3.4.38
-  vue-tsc: ^2.0.29
-  wrangler: ^3.72.1
+  vue-tsc: ^2.1.2
+  wrangler: ^3.73.0

packages/shiki/test/engine-js/verify.test.ts

@@ -0,0 +1,173 @@
+import fs from 'node:fs/promises'
+import process from 'node:process'
+import { bundledLanguages, createHighlighter, createJavaScriptRegexEngine } from 'shiki'
+import c from 'picocolors'
+import { version } from '../package.json'
+
+const engine = createJavaScriptRegexEngine()
+
+export interface ReportItem {
+  lang: string
+  highlightMatch: boolean | 'error'
+  patternsParsable: number
+  patternsFailed: [string, unknown][]
+  highlightA?: string
+  highlightB?: string
+}
+
+async function run() {
+  const report: ReportItem[] = []
+
+  for (const lang of Object.keys(bundledLanguages)) {
+    const sample = await fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')
+      .catch(() => '')
+
+    if (!sample) {
+      console.log(c.dim(`[${lang}] Sample not found`))
+      continue
+    }
+
+    let shiki = null
+    const parsablePatterns: string[] = []
+    const unparsablePatterns: [string, unknown][] = []
+
+    const shikiWasm = await createHighlighter({
+      langs: [lang],
+      themes: ['vitesse-dark'],
+    })
+
+    const grammar = shikiWasm.getLanguage(lang) as any
+    const patterns = getPatternsOfGrammar(grammar._grammar)
+    let highlightMatch: boolean | 'error' = false
+
+    for (const pattern of patterns) {
+      try {
+        engine.createScanner([pattern])
+        parsablePatterns.push(pattern)
+      }
+      catch (e: any) {
+        unparsablePatterns.push([pattern, String(e.cause || e)])
+      }
+    }
+
+    const highlightA = shikiWasm.codeToHtml(sample, { lang, theme: 'vitesse-dark' })
+    let highlightB: string | undefined
+
+    try {
+      shiki = await createHighlighter({
+        langs: [lang],
+        themes: ['vitesse-dark'],
+        engine,
+      })
+
+      highlightB = shiki.codeToHtml(sample, { lang, theme: 'vitesse-dark' })
+
+      highlightMatch = highlightA === highlightB
+
+      if (!highlightMatch) {
+        console.log(c.yellow(`[${lang}] Mismatch`))
+      }
+      else {
+        console.log(c.green(`[${lang}] OK`))
+      }
+    }
+    catch (e) {
+      highlightMatch = 'error'
+      console.log(c.red(`[${lang}] Error ${e}`))
+    }
+    finally {
+      report.push({
+        lang,
+        highlightMatch,
+        patternsParsable: parsablePatterns.length,
+        patternsFailed: unparsablePatterns,
+        ...highlightMatch === true
+          ? {}
+          : {
+              highlightA,
+              highlightB,
+            },
+      })
+
+      shikiWasm?.dispose()
+      shiki?.dispose()
+    }
+  }
+
+  const order = [true, false, 'error']
+
+  report
+    .sort((a, b) => {
+      const aOrder = order.indexOf(a.highlightMatch)
+      const bOrder = order.indexOf(b.highlightMatch)
+
+      if (aOrder !== bOrder)
+        return aOrder - bOrder
+
+      return (a.patternsFailed.length - b.patternsFailed.length) || a.lang.localeCompare(b.lang)
+    })
+
+  await fs.writeFile(
+    new URL('./report-engine-js-compat.json', import.meta.url),
+    JSON.stringify(report, null, 2),
+  )
+
+  const table: readonly [string, string, string, string][] = [
+    ['Language', 'Highlight Match', 'Patterns Parsable', 'Patterns Failed'],
+    ['---', ':---', '---', '---'],
+    ...report
+      .map(item => [
+        item.lang,
+        item.highlightMatch === true ? '✅ OK' : item.highlightMatch === 'error' ? '❌ Error' : '⚠️ Mismatch',
+        item.patternsParsable === 0 ? '-' : item.patternsParsable.toString(),
+        item.patternsFailed.length === 0 ? '-' : item.patternsFailed.length.toString(),
+      ] as [string, string, string, string]),
+  ]
+
+  const markdown = [
+    '# Report: JavaScript RegExp Engine Compatibility',
+    '',
+    `> At ${new Date().toDateString()}`,
+    '>',
+    `> Version \`${version}\``,
+    '>',
+    `> Runtime: Node.js v${process.versions.node}`,
+    '',
+    '| Status | Number |',
+    '| :--- | ---: |',
+    `| Total Languages | ${report.length} |`,
+    `| OK | ${report.filter(item => item.highlightMatch === true).length} |`,
+    `| Mismatch | ${report.filter(item => item.highlightMatch === false).length} |`,
+    `| Error | ${report.filter(item => item.highlightMatch === 'error').length} |`,
+    '',
+    table.map(row => `| ${row.join(' | ')} |`).join('\n'),
+  ].join('\n')
+  await fs.writeFile(
+    new URL('./report-engine-js-compat.md', import.meta.url),
+    markdown,
+  )
+}
+
+function getPatternsOfGrammar(grammar: any) {
+  const patterns = new Set<string>()
+
+  const scan = (obj: any) => {
+    if (!obj)
+      return
+    if (typeof obj.match === 'string')
+      patterns.add(obj.match)
+    if (typeof obj.begin === 'string')
+      patterns.add(obj.begin)
+    if (typeof obj.end === 'string')
+      patterns.add(obj.end)
+    if (obj.patterns)
+      obj.patterns.forEach(scan)
+    Object.values(obj.repository || {}).forEach(scan)
+  }
+
+  scan(grammar)
+
+  return patterns
+}
+
+run()