README.md

@@ -13,6 +13,8 @@ A beautiful syntax highlighter based on TextMate grammar, accurate and powerful.
 > [!NOTE]
 > You are viewing the **v1 branch**, which is a [major rewrite](https://github.com/shikijs/shiki/releases/tag/v1.0.0) from v0.x. Check out the [Migration Guide](https://shiki.style/guide/migrate#migrate-from-v0-14) if you are upgrading.
 >
+> Read more about [The Evolution of Shiki v1.0](https://nuxt.com/blog/shiki-v1).
+>
 > For legacy code, check the [v0.x branch](https://github.com/shikijs/shiki/tree/v0).
 
 ## License

docs/.vitepress/theme/index.ts

@@ -2,6 +2,7 @@
 import Theme from 'vitepress/theme'
 import type { EnhanceAppContext } from 'vitepress'
 import { createPinia } from 'pinia'
+import { h } from 'vue'
 import TwoslashFloatingVue from '../../../packages/vitepress-twoslash/src/client'
 
 import '../../../packages/twoslash/style-rich.css'
@@ -11,10 +12,23 @@ import 'uno.css'
 import './style.css'
 import './transformers.css'
 
+// @unocss-include
+
 export default {
   extends: Theme,
   enhanceApp({ app }: EnhanceAppContext) {
     app.use(createPinia())
     app.use(TwoslashFloatingVue)
   },
+  Layout() {
+    return h(Theme.Layout, null, {
+      'home-hero-actions-after': () => h('div', { class: 'mt-10 mb--4 vp-doc' }, [
+        h('a', { href: 'https://nuxt.com/blog/shiki-v1', target: '_blank', class: 'no-underline! flex-inline gap-1 items-center' }, [
+          h('div', { class: 'i-ph-books-duotone text-2xl' }),
+          'The Evolution of Shiki v1.0',
+          h('div', { class: 'i-ph-arrow-up-right mt--3 ml--1' }),
+        ]),
+      ]),
+    })
+  },
 }

docs/package.json

@@ -7,18 +7,19 @@
     "docs:preview": "vitepress preview"
   },
   "dependencies": {
+    "@iconify-json/ph": "^1.1.11",
     "fuse.js": "^7.0.0"
   },
   "devDependencies": {
     "@iconify-json/svg-spinners": "^1.1.2",
     "@shikijs/transformers": "workspace:*",
     "@shikijs/twoslash": "workspace:*",
-    "@unocss/reset": "^0.58.6",
+    "@unocss/reset": "^0.58.8",
     "@vueuse/core": "^10.9.0",
     "floating-vue": "^5.2.2",
     "pinia": "^2.1.7",
     "shiki": "workspace:*",
-    "unocss": "^0.58.6",
+    "unocss": "^0.58.8",
     "unplugin-vue-components": "^0.26.0",
     "vitepress": "^1.0.1",
     "vue": "^3.4.21"

docs/packages/transformers.md

@@ -111,7 +111,7 @@ console.log('Highlighted') // [!code highlight]
 console.log('Not highlighted')
 ```
 
-- `// [!code highlight]` outputs: `<span class="highlighted">`
+- `// [!code highlight]` outputs: `<span class="line highlighted">`
 - The outer `<pre>` tag is modified: `<pre class="has-highlighted">`
 
 You can also highlight multiple lines with a single comment:
@@ -198,8 +198,8 @@ console.log('Focused') // [!code focus]
 console.log('Not focused')
 ```
 
-- Outputs: `<span class="line has-focus">`
-- The outer `<pre>` tag is modified: `<pre class="has-focused-lines">`
+- Outputs: `<span class="line focused">`
+- The outer `<pre>` tag is modified: `<pre class="has-focused">`
 
 You can also focus multiple lines with a single comment:
 
@@ -235,9 +235,9 @@ console.warn('Warning') // [\!code warning]
 ```
 ````
 
-- Outputs: `<span class="highlighted error">` for errors
-- Outputs: `<span class="highlighted warning">` for warnings
-- The outer `<pre>` tag is modified: `<pre class="has-error-levels">`
+- Outputs: `<span class="line highlighted error">` for errors
+- Outputs: `<span class="line highlighted warning">` for warnings
+- The outer `<pre>` tag is modified: `<pre class="has-highlighted">`
 
 With some additional CSS rules, you can make it look like this:
 

package.json

@@ -1,6 +1,6 @@
 {
   "type": "module",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "private": true,
   "packageManager": "pnpm@8.15.5",
   "scripts": {
@@ -14,7 +14,7 @@
     "prepare": "simple-git-hooks"
   },
   "devDependencies": {
-    "@antfu/eslint-config": "^2.9.0",
+    "@antfu/eslint-config": "^2.11.6",
     "@antfu/ni": "^0.21.12",
     "@antfu/utils": "^0.7.7",
     "@rollup/plugin-alias": "^5.1.0",
@@ -30,7 +30,7 @@
     "@shikijs/vitepress-twoslash": "workspace:*",
     "@types/fs-extra": "^11.0.4",
     "@types/hast": "^3.0.4",
-    "@types/node": "^20.11.30",
+    "@types/node": "^20.12.2",
     "@vitest/coverage-v8": "^1.4.0",
     "ansi-sequence-parser": "^1.1.1",
     "bumpp": "^9.4.0",
@@ -50,7 +50,7 @@
     "pnpm": "^8.15.5",
     "prettier": "^3.2.5",
     "rimraf": "^5.0.5",
-    "rollup": "^4.13.0",
+    "rollup": "^4.13.2",
     "rollup-plugin-copy": "^3.5.0",
     "rollup-plugin-dts": "^6.1.0",
     "rollup-plugin-esbuild": "^6.1.1",
@@ -60,12 +60,12 @@
     "taze": "^0.13.3",
     "typescript": "^5.4.3",
     "unbuild": "^2.0.0",
-    "vite": "^5.2.6",
+    "vite": "^5.2.7",
     "vite-tsconfig-paths": "^4.3.2",
     "vitepress-plugin-mermaid": "^2.0.16",
     "vitest": "^1.4.0",
     "vue-tsc": "^2.0.7",
-    "wrangler": "^3.37.0"
+    "wrangler": "^3.41.0"
   },
   "resolutions": {
     "@shikijs/compat": "workspace:*",

packages/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/cli",
   "type": "module",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Shiki in the command line",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/compat/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/compat",
   "type": "module",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Shiki v0.x compatible API",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/markdown-it/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/markdown-it",
   "type": "module",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "markdown-it integration for shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/monaco/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/monaco",
   "type": "module",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Use Shiki for Monaco Editor",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/monaco/playground/package.json

@@ -8,6 +8,6 @@
   },
   "devDependencies": {
     "typescript": "^5.4.3",
-    "vite": "^5.2.6"
+    "vite": "^5.2.7"
   }
 }

packages/monaco/src/index.ts

@@ -1,16 +1,9 @@
-import type * as monaco from 'monaco-editor-core'
+import type monacoNs from 'monaco-editor-core'
 import type { ShikiInternal, ThemeRegistrationResolved } from '@shikijs/core'
 import type { StateStack } from '@shikijs/core/textmate'
 import { INITIAL, StackElementMetadata } from '@shikijs/core/textmate'
 
-type Monaco = typeof monaco
-
-export interface MonacoInferface {
-  editor: Monaco['editor']
-  languages: Monaco['languages']
-}
-
-export interface MonacoTheme extends monaco.editor.IStandaloneThemeData { }
+export interface MonacoTheme extends monacoNs.editor.IStandaloneThemeData {}
 
 export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): MonacoTheme {
   let rules = 'rules' in theme
@@ -48,7 +41,7 @@ export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): Mo
 
 export function shikiToMonaco(
   highlighter: ShikiInternal<any, any>,
-  monaco: MonacoInferface,
+  monaco: typeof monacoNs,
 ) {
   // Convert themes to Monaco themes and register them
   const themeMap = new Map<string, MonacoTheme>()
@@ -60,21 +53,41 @@ export function shikiToMonaco(
     monaco.editor.defineTheme(themeId, monacoTheme)
   }
 
-  let currentTheme = themeIds[0]
+  const colorMap: string[] = []
+  const colorToScopeMap = new Map<string, string>()
 
   // Because Monaco does not have the API of reading the current theme,
   // We hijack it here to keep track of the current theme.
   const _setTheme = monaco.editor.setTheme.bind(monaco.editor)
-  monaco.editor.setTheme = (theme: string) => {
-    _setTheme(theme)
-    currentTheme = theme
+  monaco.editor.setTheme = (themeName: string) => {
+    const ret = highlighter.setTheme(themeName)
+    const theme = themeMap.get(themeName)
+    colorMap.length = ret.colorMap.length
+    ret.colorMap.forEach((color, i) => {
+      colorMap[i] = color
+    })
+    colorToScopeMap.clear()
+    theme?.rules.forEach((rule) => {
+      const c = normalizeColor(rule.foreground)
+      if (c && !colorToScopeMap.has(c))
+        colorToScopeMap.set(c, rule.token)
+    })
+    _setTheme(themeName)
+  }
+
+  // Set the first theme as the default theme
+  monaco.editor.setTheme(themeIds[0])
+
+  function findScopeByColor(color: string) {
+    return colorToScopeMap.get(color)
   }
 
+  const monacoLanguageIds = new Set(monaco.languages.getLanguages().map(l => l.id))
   for (const lang of highlighter.getLoadedLanguages()) {
-    if (monaco.languages.getLanguages().some(l => l.id === lang)) {
+    if (monacoLanguageIds.has(lang)) {
       monaco.languages.setTokensProvider(lang, {
         getInitialState() {
-          return new TokenizerState(INITIAL, highlighter)
+          return new TokenizerState(INITIAL)
         },
         tokenize(line, state: TokenizerState) {
           // Do not attempt to tokenize if a line is too long
@@ -89,26 +102,12 @@ export function shikiToMonaco(
             }
           }
 
-          const grammar = state.highlighter.getLanguage(lang)
-          const { colorMap } = state.highlighter.setTheme(currentTheme)
-          const theme = themeMap.get(currentTheme)
+          const grammar = highlighter.getLanguage(lang)
           const result = grammar.tokenizeLine2(line, state.ruleStack, tokenizeTimeLimit)
 
           if (result.stoppedEarly)
             console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`)
 
-          const colorToScopeMap = new Map<string, string>()
-
-          theme!.rules.forEach((rule) => {
-            const c = normalizeColor(rule.foreground)
-            if (c && !colorToScopeMap.has(c))
-              colorToScopeMap.set(c, rule.token)
-          })
-
-          function findScopeByColor(color: string) {
-            return colorToScopeMap.get(color)
-          }
-
           const tokensLength = result.tokens.length / 2
           const tokens: any[] = []
           for (let j = 0; j < tokensLength; j++) {
@@ -118,38 +117,32 @@ export function shikiToMonaco(
             // Because Monaco only support one scope per token,
             // we workaround this to use color to trace back the scope
             const scope = findScopeByColor(color) || ''
-            tokens.push({
-              startIndex,
-              scopes: scope,
-            })
+            tokens.push({ startIndex, scopes: scope })
           }
 
-          return {
-            endState: new TokenizerState(result.ruleStack, state.highlighter),
-            tokens,
-          }
+          return { endState: new TokenizerState(result.ruleStack), tokens }
         },
       })
     }
   }
 }
 
-class TokenizerState implements monaco.languages.IState {
+class TokenizerState implements monacoNs.languages.IState {
   constructor(
     private _ruleStack: StateStack,
-    public highlighter: ShikiInternal<any, any>,
-  ) { }
+  ) {}
 
   public get ruleStack(): StateStack {
     return this._ruleStack
   }
 
   public clone(): TokenizerState {
-    return new TokenizerState(this._ruleStack, this.highlighter)
+    return new TokenizerState(this._ruleStack)
   }
 
-  public equals(other: monaco.languages.IState): boolean {
-    if (!other
+  public equals(other: monacoNs.languages.IState): boolean {
+    if (
+      !other
       || !(other instanceof TokenizerState)
       || other !== this
       || other._ruleStack !== this._ruleStack
@@ -166,9 +159,12 @@ function normalizeColor(color: string | undefined): string | undefined
 function normalizeColor(color: string | undefined) {
   if (!color)
     return color
-  color = color.replace('#', '').toLowerCase()
+
+  color = (color.charCodeAt(0) === 35 ? color.slice(1) : color).toLowerCase()
+
   // #RGB => #RRGGBB - Monaco does not support hex color with 3 or 4 digits
   if (color.length === 3 || color.length === 4)
     color = color.split('').map(c => c + c).join('')
+
   return color
 }

packages/rehype/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/rehype",
   "type": "module",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "rehype integration for shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/shiki/package.json

@@ -1,7 +1,7 @@
 {
   "name": "shiki",
   "type": "module",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "A beautiful Syntax Highlighter.",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -103,7 +103,7 @@
     "@shikijs/core": "workspace:*"
   },
   "devDependencies": {
-    "tm-grammars": "^1.5.6",
+    "tm-grammars": "^1.6.0",
     "tm-themes": "^1.3.2",
     "vscode-oniguruma": "^1.7.0"
   }