.github/workflows/ci.yml

@@ -14,8 +14,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: recursive
 
       - name: Install pnpm
         uses: pnpm/action-setup@v2
@@ -38,8 +36,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: recursive
 
       - name: Install pnpm
         uses: pnpm/action-setup@v2
@@ -75,8 +71,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: recursive
 
       - name: Install pnpm
         uses: pnpm/action-setup@v3

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "packages/core/vendor/vscode-textmate"]
-	path = packages/core/vendor/vscode-textmate
-	url = https://github.com/microsoft/vscode-textmate

CONTRIBUTING.md

@@ -4,22 +4,6 @@ Thanks for lending a hand 👋
 
 ## Development
 
-### Clone
-
-This repository contains a [git submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to `vscode-textmate`. By default `git clone` does not clone submodules. To clone this repository and its submodules, use:
-
-```bash
-git clone --recursive https://github.com/shikijs/shiki
-```
-
-Or if you have already cloned it, use:
-
-```bash
-git submodule update --init --recursive
-```
-
-Learn more at this [StackOverflow thread](https://stackoverflow.com/a/4438292).
-
 ### Setup
 
 - We use [pnpm](https://pnpm.js.org/) to manage dependencies. Install it with `npm i -g pnpm`.

bench/engines.bench.ts

@@ -1,39 +1,43 @@
+/* eslint-disable no-console */
 import fs from 'node:fs/promises'
 import { bench, describe } from 'vitest'
 import type { BundledLanguage } from 'shiki'
 import { createHighlighter, createJavaScriptRegexEngine, createWasmOnigEngine } from 'shiki'
 import type { ReportItem } from '../scripts/report-engine-js-compat'
 
-describe('engines', async () => {
-  const js = createJavaScriptRegexEngine()
-  const wasm = await createWasmOnigEngine(() => import('shiki/wasm'))
+const js = createJavaScriptRegexEngine()
+const wasm = await createWasmOnigEngine(() => import('shiki/wasm'))
 
-  // Run `npx jiti scripts/report-engine-js-compat.ts` to generate the report first
-  const report = await fs.readFile('../scripts/report-engine-js-compat.json', 'utf-8').then(JSON.parse) as ReportItem[]
-  const langs = report.filter(i => i.highlightMatch === true).map(i => i.lang) as BundledLanguage[]
-  const samples = await Promise.all(langs.map(lang => fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')))
+const RANGE = [0, 20]
 
-  const shikiJs = await createHighlighter({
-    langs,
-    themes: ['vitesse-dark'],
-    engine: js,
-  })
+// Run `npx jiti scripts/report-engine-js-compat.ts` to generate the report first
+const report = await fs.readFile(new URL('../scripts/report-engine-js-compat.json', import.meta.url), 'utf-8').then(JSON.parse) as ReportItem[]
+const langs = report.filter(i => i.highlightMatch === true).map(i => i.lang).slice(...RANGE) as BundledLanguage[]
+// Clone https://github.com/shikijs/textmate-grammars-themes to `../tm-grammars-themes`
+const samples = await Promise.all(langs.map(lang => fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')))
 
-  const shikiWasm = await createHighlighter({
-    langs,
-    themes: ['vitesse-dark'],
-    engine: wasm,
-  })
+console.log('Benchmarking engines with', langs.length, 'languages')
+
+const shikiJs = await createHighlighter({
+  langs,
+  themes: ['vitesse-dark'],
+  engine: js,
+})
 
-  bench('js', () => {
-    for (const lang of langs) {
+const shikiWasm = await createHighlighter({
+  langs,
+  themes: ['vitesse-dark'],
+  engine: wasm,
+})
+
+for (const lang of langs) {
+  describe(lang, () => {
+    bench('js', () => {
       shikiJs.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
-    }
-  }, { warmupIterations: 10, iterations: 30 })
+    })
 
-  bench('wasm', () => {
-    for (const lang of langs) {
+    bench('wasm', () => {
       shikiWasm.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
-    }
-  }, { warmupIterations: 10, iterations: 30 })
-})
+    })
+  })
+}

docs/.vitepress/config.ts

@@ -11,21 +11,24 @@ import vite from './vite.config'
 
 const GUIDES: DefaultTheme.NavItemWithLink[] = [
   { text: 'Getting Started', link: '/guide/' },
-  { text: 'Installation', link: '/guide/install' },
+  { text: 'Installation & Usage', link: '/guide/install' },
   { text: 'Bundles', link: '/guide/bundles' },
   { text: 'Dual Themes', link: '/guide/dual-themes' },
   { text: 'Decorations', link: '/guide/decorations' },
   { text: 'Transformers', link: '/guide/transformers' },
   { text: 'Theme Colors Manipulation', link: '/guide/theme-colors' },
-  { text: 'Migration', link: '/guide/migrate' },
-  { text: 'Compatibility Build', link: '/guide/compat' },
+  { text: 'RegExp Engines', link: '/guide/regex-engines' },
+  { text: 'Synchronous Usage', link: '/guide/sync-usage' },
   { text: 'Custom Themes', link: '/guide/load-theme' },
   { text: 'Custom Languages', link: '/guide/load-lang' },
+  { text: 'Migration', link: '/guide/migrate' },
+  { text: 'Compatibility Build', link: '/guide/compat' },
 ]
 
 const REFERENCES: DefaultTheme.NavItemWithLink[] = [
   { text: 'Themes', link: '/themes' },
   { text: 'Languages', link: '/languages' },
+  { text: 'JavaScript Engine Compatibility', link: '/references/engine-js-compat' },
 ]
 
 const INTEGRATIONS: DefaultTheme.NavItemWithLink[] = [

docs/guide/bundles.md

@@ -4,17 +4,19 @@ outline: deep
 
 # Bundles
 
-The main `shiki` entries bundles all supported themes and languages via lazy dynamic imports. The efficiency shouldn't be a concern to most of the scenarios as the grammar would only be imported/downloaded when it is used. However, when you bundle Shiki into browsers runtime or web workers, even those files are not imported, they still add up to your dist size. We provide the [fine-grained bundle](/guide/install#fine-grained-bundle) to help you compose languages and themes one-by-one as you need.
+The main `shiki` entries bundles all supported themes and languages via lazy dynamic imports. The efficiency shouldn't be a concern to most of the scenarios as the grammar would only be imported/downloaded when it is used. However, when you bundle Shiki into browsers runtime or web workers, even those files are not imported, they still add up to your dist size. We provide the [fine-grained bundle](#fine-grained-bundle) to help you compose languages and themes one-by-one as you need.
+
+## Bundle Presets
 
 To make it easier, we also provide some pre-composed bundles for you to use:
 
-## `shiki/bundle/full`
+### `shiki/bundle/full`
 
 > [Bundle Size](/guide/#bundle-size): 6.4 MB (minified), 1.2 MB (gzip), async chunks included
 
 The full bundle includes all themes and languages, same as the main `shiki` entry.
 
-## `shiki/bundle/web`
+### `shiki/bundle/web`
 
 > [Bundle Size](/guide/#bundle-size): 3.8 MB (minified), 695 KB (gzip), async chunks included
 
@@ -35,3 +37,46 @@ const highlighter = await createHighlighter({
   themes: ['github-dark', 'github-light'],
 })
 ```
+
+## Fine-grained Bundle
+
+When importing `shiki`, all the themes and languages are bundled as async chunks. Normally it won't be a concern to you as they are not being loaded if you don't use them. In some cases, if you want to control what to bundle, you can use the core and compose your own bundle.
+
+```ts twoslash
+// @noErrors
+// `shiki/core` entry does not include any themes or languages or the wasm binary.
+import { createHighlighterCore } from 'shiki/core'
+
+// directly import the theme and language modules, only the ones you imported will be bundled.
+import nord from 'shiki/themes/nord.mjs'
+
+const highlighter = await createHighlighterCore({
+  themes: [
+    // instead of strings, you need to pass the imported module
+    nord,
+    // or a dynamic import if you want to do chunk splitting
+    import('shiki/themes/material-theme-ocean.mjs')
+  ],
+  langs: [
+    import('shiki/langs/javascript.mjs'),
+    // shiki will try to interop the module with the default export
+    () => import('shiki/langs/css.mjs'),
+    // or a getter that returns custom grammar
+    async () => JSON.parse(await fs.readFile('my-grammar.json', 'utf-8'))
+  ],
+  // `shiki/wasm` contains the wasm binary inlined as base64 string.
+  engine: createWasmOnigEngine(import('shiki/wasm'))
+})
+
+// optionally, load themes and languages after creation
+await highlighter.loadTheme(import('shiki/themes/vitesse-light.mjs'))
+
+const code = highlighter.codeToHtml('const a = 1', {
+  lang: 'javascript',
+  theme: 'material-theme-ocean'
+})
+```
+
+::: info
+[Shorthands](/guide/install#shorthands) are only avaliable in bundle presets. For a fine-grained bundle, you can create your own shorthands using [`createSingletonShorthands`](https://github.com/shikijs/shiki/blob/main/packages/core/src/constructors/bundle-factory.ts#L203) or port it yourself.
+:::

docs/guide/index.md

@@ -17,7 +17,7 @@ Oh by the way, all the code blocks in this site are highlighted by Shiki, as you
 - All grammars/themes/wasm served as ESM, lazy-loaded on demand and bundler-friendly.
 - Portable & agnostic. Does not rely on Node.js APIs or the filesystem, works in any modern JavaScript runtime.
 - ESM-only ([CDN Usage](/guide/install#cdn-usage), [CJS Usage](/guide/install#cjs-usage)).
-- [Bundles languages/themes composedly](/guide/install#fine-grained-bundle).
+- [Bundles languages/themes composedly](/guide/bundles#fine-grained-bundle).
 - [Light/Dark themes support](/guide/dual-themes)
 - [`hast` support](/guide/transformers#codetohast)
 - [Transformers API](/guide/transformers)

docs/guide/install.md

@@ -2,10 +2,12 @@
 outline: deep
 ---
 
-# Installation
+# Installation & Usage
 
 <Badges name="shiki" />
 
+## Installation
+
 Install via npm, or see [CDN Usage](#cdn-usage):
 ::: code-group
 
@@ -160,46 +162,7 @@ highlighter.codeToHtml('const a = 1', {
 
 When importing `shiki`, all the themes and languages are bundled as async chunks. Normally it won't be a concern to you as they are not being loaded if you don't use them. In some cases, if you want to control what to bundle, you can use the core and compose your own bundle.
 
-```ts twoslash theme:material-theme-ocean
-// @noErrors
-// `shiki/core` entry does not include any themes or languages or the wasm binary.
-import { createHighlighterCore } from 'shiki/core'
-
-// `shiki/wasm` contains the wasm binary inlined as base64 string.
-import getWasm from 'shiki/wasm'
-
-// directly import the theme and language modules, only the ones you imported will be bundled.
-import nord from 'shiki/themes/nord.mjs'
-
-const highlighter = await createHighlighterCore({
-  themes: [
-    // instead of strings, you need to pass the imported module
-    nord,
-    // or a dynamic import if you want to do chunk splitting
-    import('shiki/themes/material-theme-ocean.mjs')
-  ],
-  langs: [
-    import('shiki/langs/javascript.mjs'),
-    // shiki will try to interop the module with the default export
-    () => import('shiki/langs/css.mjs'),
-    // or a getter that returns custom grammar
-    async () => JSON.parse(await fs.readFile('my-grammar.json', 'utf-8'))
-  ],
-  loadWasm: getWasm
-})
-
-// optionally, load themes and languages after creation
-await highlighter.loadTheme(import('shiki/themes/vitesse-light.mjs'))
-
-const code = highlighter.codeToHtml('const a = 1', {
-  lang: 'javascript',
-  theme: 'material-theme-ocean'
-})
-```
-
-::: info
-[Shorthands](#shorthands) are only avaliable in [bundled usage](#bundled-usage). For a fine-grained bundle, you can create your own shorthands using [`createSingletonShorthands`](https://github.com/shikijs/shiki/blob/main/packages/core/src/bundle-factory.ts) or port it yourself.
-:::
+Check out the [Fine-grained Bundle](/guide/bundles#fine-grained-bundle) section for more details.
 
 ### Bundle Presets
 

docs/guide/regex-engines.md

@@ -0,0 +1,57 @@
+---
+outline: deep
+---
+
+# RegExp Engines
+
+TextMate grammars is based on regular expressions to match tokens. Usually, we use [Oniguruma](https://github.com/kkos/oniguruma) (a regular expression engine written in C) to parse the grammar. To make it work in JavaScript, we compile Oniguruma to WebAssembly to run in the browser or Node.js.
+
+Since v1.15, we expose the ability to for users to switch the RegExp engine and provide custom implementations.
+
+An `engine` option is added to the `createHighlighter` and `createHighlighterCore`. For example:
+
+```ts
+import { createHighlighter } from 'shiki'
+
+const shiki = await createShiki({
+  themes: ['nord'],
+  langs: ['javascript'],
+  engine: { /* custom engine */ }
+})
+```
+
+Shiki come with two built-in engines:
+
+## Oniguruma Engine
+
+This is the default engine that uses the compiled Oniguruma WebAssembly. The most accurate and robust engine.
+
+```ts
+import { createHighlighter, createWasmOnigEngine } from 'shiki'
+
+const shiki = await createShiki({
+  themes: ['nord'],
+  langs: ['javascript'],
+  engine: createWasmOnigEngine(import('shiki/wasm'))
+})
+```
+
+## JavaScript RegExp Engine (Experimental)
+
+This experimental engine uses JavaScript's native RegExp. As TextMate grammars' regular expressions are in Oniguruma flavor that might contains syntaxes that are not supported by JavaScript's RegExp, we use [`oniguruma-to-js`](https://github.com/antfu/oniguruma-to-js) to lowering the syntaxes and try to make them compatible with JavaScript's RegExp.
+
+Please check the [compatibility table](/references/engine-js-compat) to check the support status of the languages you are using.
+
+```ts {3,8}
+import { createHighlighter, createJavaScriptRegexEngine } from 'shiki'
+
+const jsEngine = createJavaScriptRegexEngine()
+
+const shiki = await createHighlighter({
+  themes: ['nord'],
+  langs: ['javascript'],
+  engine: jsEngine
+})
+
+const html = shiki.codeToHtml('const a = 1', { lang: 'javascript', theme: 'nord' })
+```

docs/guide/sync-usage.md

@@ -0,0 +1,40 @@
+# Synchronous Usage
+
+The `await createHighlighter()` and `highlighter.codeToHtml()` are already the effort to do the seperations of asynchronism and synchronism. For most of the cases, you should be able to resolve the async part in the initialization phase and use the highlighter synchronously later.
+
+In some extreme cases that you need to run Shiki completely synchronously, since v1.16, we provide a synchronous version of the core API. You can use `createHighlighterCoreSync` to create a highlighter instance synchronously.
+
+```ts
+import { createHighlighterCoreSync, createJavaScriptRegexEngine } from 'shiki/core'
+import js from 'shiki/core/langs/javascript.mjs'
+import nord from 'shiki/core/themes/nord.mjs'
+
+const shiki = createHighlighterCoreSync({
+  themes: [nord],
+  langs: [js],
+  engine: createJavaScriptRegexEngine()
+})
+
+const html = shiki.highlight('console.log(1)', { lang: 'js', theme: 'nord' })
+```
+
+When doing so, it requires all `themes` and `langs` to be provide as plain objects. Also an explicit `engine` is required to be provided. With the new [experimental JavaScript RegExp engine](http://localhost:5173/guide/regex-engines#javascript-regexp-engine-experimental) you are able to create an engine instance synchronously as well.
+
+The [Oniguruma Engine](http://localhost:5173/guide/regex-engines#oniguruma-engine) can only be created asynchronously, so you need to resolve the engine promise before creating the sync highlighter.
+
+```ts
+import { createHighlighterCoreSync, createWasmOnigEngine } from 'shiki/core'
+import js from 'shiki/core/langs/javascript.mjs'
+import nord from 'shiki/core/themes/nord.mjs'
+
+// Load this somewhere beforehand
+const engine = await createWasmOnigEngine(import('shiki/wasm'))
+
+const shiki = createHighlighterCoreSync({
+  themes: [nord],
+  langs: [js],
+  engine, // if a resolved engine passed in, the rest can still be synced.
+})
+
+const html = shiki.highlight('console.log(1)', { lang: 'js', theme: 'nord' })
+```