docs/package.json

@@ -17,7 +17,7 @@
     "@unocss/reset": "^0.61.5",
     "@vueuse/core": "^10.11.0",
     "floating-vue": "^5.2.2",
-    "pinia": "^2.1.7",
+    "pinia": "^2.2.0",
     "shiki": "workspace:*",
     "unocss": "^0.61.5",
     "unplugin-vue-components": "^0.27.3",

package.json

@@ -1,6 +1,6 @@
 {
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "private": true,
   "packageManager": "pnpm@9.6.0",
   "scripts": {
@@ -64,7 +64,7 @@
     "vitepress-plugin-mermaid": "^2.0.16",
     "vitest": "^2.0.4",
     "vue-tsc": "^2.0.29",
-    "wrangler": "^3.67.0"
+    "wrangler": "^3.67.1"
   },
   "resolutions": {
     "@shikijs/compat": "workspace:*",

packages/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/cli",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "Shiki in the command line",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/compat/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/compat",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "Shiki v0.x compatible API",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/core/src/grammar-state.ts

@@ -1,17 +1,25 @@
 import type { StateStackImpl } from '../vendor/vscode-textmate/src/grammar'
 import { ShikiError } from './error'
 import type { StateStack } from './textmate'
+import { INITIAL } from './textmate'
 
 /**
  * GrammarState is a special reference object that holds the state of a grammar.
  *
  * It's used to highlight code snippets that are part of the target language.
  */
 export class GrammarState {
+  /**
+   * Static method to create a initial grammar state.
+   */
+  static initial(lang: string, theme: string) {
+    return new GrammarState(INITIAL, lang, theme)
+  }
+
   constructor(
-    private _stack: StateStack,
-    public lang: string,
-    public theme: string,
+    private readonly _stack: StateStack,
+    public readonly lang: string,
+    public readonly theme: string,
   ) {}
 
   get scopes() {

packages/core/src/transformer-decorations.ts

@@ -21,12 +21,21 @@ export function transformerDecorations(): ShikiTransformer {
 
       function normalizePosition(p: OffsetOrPosition): ResolvedPosition {
         if (typeof p === 'number') {
+          if (p < 0 || p > shiki.source.length)
+            throw new ShikiError(`Invalid decoration offset: ${p}. Code length: ${shiki.source.length}`)
+
           return {
             ...converter.indexToPos(p),
             offset: p,
           }
         }
         else {
+          const line = converter.lines[p.line]
+          if (line === undefined)
+            throw new ShikiError(`Invalid decoration position ${JSON.stringify(p)}. Lines length: ${converter.lines.length}`)
+          if (p.character < 0 || p.character > line.length)
+            throw new ShikiError(`Invalid decoration position ${JSON.stringify(p)}. Line ${p.line} length: ${line.length}`)
+
           return {
             ...p,
             offset: converter.posToIndex(p.line, p.character),

packages/core/src/types/decorations.ts

@@ -14,8 +14,6 @@ export interface DecorationItem {
   start: OffsetOrPosition
   /**
    * End offset or position of the decoration.
-   *
-   * If the
    */
   end: OffsetOrPosition
   /**

packages/core/src/utils.ts

@@ -7,7 +7,7 @@ export function toArray<T>(x: MaybeArray<T>): T[] {
 }
 
 /**
- * Slipt a string into lines, each line preserves the line ending.
+ * Split a string into lines, each line preserves the line ending.
  */
 export function splitLines(code: string, preserveEnding = false): [string, number][] {
   const parts = code.split(/(\r?\n)/g)
@@ -192,11 +192,20 @@ export function stringifyTokenStyle(token: Record<string, string>) {
 
 /**
  * Creates a converter between index and position in a code block.
+ *
+ * Overflow/underflow are unchecked.
  */
 export function createPositionConverter(code: string) {
   const lines = splitLines(code, true).map(([line]) => line)
 
   function indexToPos(index: number): Position {
+    if (index === code.length) {
+      return {
+        line: lines.length - 1,
+        character: lines[lines.length - 1].length,
+      }
+    }
+
     let character = index
     let line = 0
     for (const lineText of lines) {

packages/markdown-it/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/markdown-it",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "markdown-it integration for shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/monaco/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/monaco",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "Use Shiki for Monaco Editor",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/rehype/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/rehype",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "rehype integration for shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/shiki/package.json

@@ -1,7 +1,7 @@
 {
   "name": "shiki",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "A beautiful Syntax Highlighter.",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -104,7 +104,7 @@
     "@types/hast": "^3.0.4"
   },
   "devDependencies": {
-    "tm-grammars": "^1.14.0",
+    "tm-grammars": "^1.15.1",
     "tm-themes": "^1.5.7",
     "vscode-oniguruma": "^1.7.0"
   }

packages/shiki/src/assets/langs-bundle-full.ts

@@ -1085,6 +1085,14 @@ export const bundledLanguagesInfo: BundledLanguageInfo[] = [
     'name': 'TOML',
     'import': (() => import('./langs/toml')) as DynamicImportLanguageRegistration
   },
+  {
+    'id': 'ts-tags',
+    'name': 'TypeScript with Tags',
+    'aliases': [
+      'lit'
+    ],
+    'import': (() => import('./langs/ts-tags')) as DynamicImportLanguageRegistration
+  },
   {
     'id': 'tsv',
     'name': 'TSV',
@@ -1401,6 +1409,7 @@ export type BundledLanguage =
   | 'less'
   | 'liquid'
   | 'lisp'
+  | 'lit'
   | 'log'
   | 'logo'
   | 'lua'
@@ -1505,6 +1514,7 @@ export type BundledLanguage =
   | 'tfvars'
   | 'toml'
   | 'ts'
+  | 'ts-tags'
   | 'tsp'
   | 'tsv'
   | 'tsx'

packages/shiki/src/assets/langs-bundle-web.ts

@@ -274,6 +274,14 @@ export const bundledLanguagesInfo: BundledLanguageInfo[] = [
     'name': 'TOML',
     'import': (() => import('./langs/toml')) as DynamicImportLanguageRegistration
   },
+  {
+    'id': 'ts-tags',
+    'name': 'TypeScript with Tags',
+    'aliases': [
+      'lit'
+    ],
+    'import': (() => import('./langs/ts-tags')) as DynamicImportLanguageRegistration
+  },
   {
     'id': 'tsx',
     'name': 'TSX',
@@ -362,6 +370,7 @@ export type BundledLanguage =
   | 'jsx'
   | 'julia'
   | 'less'
+  | 'lit'
   | 'lua'
   | 'markdown'
   | 'marko'
@@ -389,6 +398,7 @@ export type BundledLanguage =
   | 'svelte'
   | 'toml'
   | 'ts'
+  | 'ts-tags'
   | 'tsx'
   | 'typescript'
   | 'vue'

packages/shiki/test/bundle.test.ts

@@ -7,7 +7,7 @@ it('bundle-full', async () => {
   }))
 
   expect(highlighter.getLoadedLanguages().length)
-    .toMatchInlineSnapshot(`299`)
+    .toMatchInlineSnapshot(`306`)
 })
 
 it('bundle-web', async () => {
@@ -17,5 +17,5 @@ it('bundle-web', async () => {
   }))
 
   expect(highlighter.getLoadedLanguages().length)
-    .toMatchInlineSnapshot(`84`)
+    .toMatchInlineSnapshot(`91`)
 })

packages/shiki/test/decorations.test.ts

@@ -23,7 +23,7 @@ export function codeToHtml(
   let result = hastToHtml(codeToHast(internal, code, options, context))
   return result
 }
-`
+// final`
 
 describe('decorations', () => {
   it('works', async () => {
@@ -78,6 +78,13 @@ describe('decorations', () => {
           end: { line: 8, character: 25 },
           properties: { class: 'highlighted' },
         },
+        // "// final"
+        // Testing offset === code.length edge case
+        {
+          start: code.length - 8,
+          end: code.length,
+          properties: { class: 'highlighted' },
+        },
       ],
     })
 
@@ -126,7 +133,7 @@ describe('decorations errors', () => {
         ],
       })
     }).rejects
-      .toThrowErrorMatchingInlineSnapshot(`[TypeError: Cannot read properties of undefined (reading 'length')]`)
+      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":100,"character":0}. Lines length: 12]`)
   })
 
   it('throws when chars overflow', async () => {
@@ -139,6 +146,40 @@ describe('decorations errors', () => {
         ],
       })
     }).rejects
-      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Failed to find end index for decoration {"line":0,"character":10,"offset":10}]`)
+      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":0,"character":10}. Line 0 length: 4]`)
+
+    expect(async () => {
+      await codeToHtml(code, {
+        theme: 'vitesse-light',
+        lang: 'ts',
+        decorations: [
+          {
+            start: { line: 2, character: 1 },
+            end: { line: 1, character: 36 }, // actual position is { line: 2, character: 3, offset 40 }
+          },
+        ],
+      })
+    }).rejects
+      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":1,"character":36}. Line 1 length: 33]`)
+  })
+
+  it('throws when offset underflows/overflows', async () => {
+    expect(async () => {
+      await codeToHtml(code, {
+        theme: 'vitesse-light',
+        lang: 'ts',
+        decorations: [{ start: 1, end: 1000 }],
+      })
+    }).rejects
+      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: 1000. Code length: 252]`)
+
+    expect(async () => {
+      await codeToHtml(code, {
+        theme: 'vitesse-light',
+        lang: 'ts',
+        decorations: [{ start: -3, end: 5 }],
+      })
+    }).rejects
+      .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: -3. Code length: 252]`)
   })
 })

packages/transformers/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/transformers",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "Collective of common transformers transformers for Shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/twoslash/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/twoslash",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "Shiki transformer for twoslash",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

packages/vitepress-twoslash/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/vitepress-twoslash",
   "type": "module",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "Enable Twoslash support in VitePress",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",