shikijs · Aug 30, 2024 · Aug 30, 2024 · Aug 30, 2024 · Aug 30, 2024 · Aug 30, 2024
diff --git a/.gitignore b/.gitignore
@@ -16,3 +16,4 @@ packages/shiki/src/assets/*.json
 cache
 .eslintcache
 report-engine-js-compat.json
+scripts/compares
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "private": true,
   "packageManager": "pnpm@9.9.0",
   "scripts": {
@@ -11,6 +11,8 @@
     "build": "pnpm -r run build",
     "docs": "pnpm -C docs run docs:dev",
     "docs:build": "pnpm -C docs run docs:build",
+    "report-engine-js": "jiti scripts/report-engine-js-compat.ts",
+    "bench": "vitest bench",
     "prepare": "simple-git-hooks"
   },
   "devDependencies": {
@@ -39,6 +41,7 @@
     "esno": "catalog:",
     "fast-glob": "catalog:",
     "fs-extra": "catalog:",
+    "jiti": "catalog:",
     "jsonc-parser": "catalog:",
     "lint-staged": "catalog:",
     "markdown-it": "catalog:",

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/cli",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "Shiki in the command line",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/packages/compat/package.json b/packages/compat/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/compat",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "Shiki v0.x compatible API",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/packages/core/src/engines/javascript.ts b/packages/core/src/engines/javascript.ts
@@ -25,9 +25,14 @@ export class JavaScriptScanner implements PatternScanner {
       try {
         const regex = onigurumaToRegexp(
           p
+            .replace(/\|\\G(\||\))/g, '$1')
+            .replace(/(\(|\|)\\G\|/g, '$1')
             // YAML specific handling; TODO: move to tm-grammars
             .replaceAll('[^\\s[-?:,\\[\\]{}#&*!|>\'"%@`]]', '[^\\s\\-?:,\\[\\]{}#&*!|>\'"%@`]'),
-          { flags: 'dg' },
+          {
+            flags: 'dgm',
+            ignoreContiguousAnchors: true,
+          },
         )
         cache?.set(p, regex)
         return regex

diff --git a/packages/markdown-it/package.json b/packages/markdown-it/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/markdown-it",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "markdown-it integration for shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/packages/monaco/package.json b/packages/monaco/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/monaco",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "Use Shiki for Monaco Editor",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/packages/rehype/package.json b/packages/rehype/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/rehype",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "rehype integration for shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/packages/shiki/package.json b/packages/shiki/package.json
@@ -1,7 +1,7 @@
 {
   "name": "shiki",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "A beautiful Syntax Highlighter.",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/packages/shiki/test/engine-js/__records__/jsonc.json b/packages/shiki/test/engine-js/__records__/jsonc.json
@@ -0,0 +1,241 @@
+[
+  {
+    "constractor": [
+      [
+        "\\b(?:true|false|null)\\b",
+        "-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?",
+        "\"",
+        "\\[",
+        "\\{",
+        "/\\*\\*(?!/)",
+        "/\\*",
+        "(//).*$\\n?"
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "// comment\n",
+          0
+        ],
+        "result": {
+          "index": 7,
+          "captureIndices": [
+            {
+              "start": 0,
+              "end": 11,
+              "length": 11
+            },
+            {
+              "start": 0,
+              "end": 2,
+              "length": 2
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "// comment\n",
+          11
+        ],
+        "result": null
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          0
+        ],
+        "result": {
+          "index": 4,
+          "captureIndices": [
+            {
+              "start": 0,
+              "end": 1,
+              "length": 1
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          13
+        ],
+        "result": null
+      }
+    ]
+  },
+  {
+    "constractor": [
+      [
+        "\\}",
+        "\"",
+        "/\\*\\*(?!/)",
+        "/\\*",
+        "(//).*$\\n?",
+        ":",
+        "[^\\s\\}]"
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          1
+        ],
+        "result": {
+          "index": 1,
+          "captureIndices": [
+            {
+              "start": 1,
+              "end": 2,
+              "length": 1
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          6
+        ],
+        "result": {
+          "index": 5,
+          "captureIndices": [
+            {
+              "start": 6,
+              "end": 7,
+              "length": 1
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          12
+        ],
+        "result": {
+          "index": 0,
+          "captureIndices": [
+            {
+              "start": 12,
+              "end": 13,
+              "length": 1
+            }
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "constractor": [
+      [
+        "\"",
+        "\\\\(?:[\"\\\\/bfnrt]|u[0-9a-fA-F]{4})",
+        "\\\\."
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          2
+        ],
+        "result": {
+          "index": 0,
+          "captureIndices": [
+            {
+              "start": 5,
+              "end": 6,
+              "length": 1
+            }
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "constractor": [
+      [
+        "(,)|(?=\\})",
+        "\\b(?:true|false|null)\\b",
+        "-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?",
+        "\"",
+        "\\[",
+        "\\{",
+        "/\\*\\*(?!/)",
+        "/\\*",
+        "(//).*$\\n?",
+        "[^\\s,]"
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          7
+        ],
+        "result": {
+          "index": 3,
+          "captureIndices": [
+            {
+              "start": 7,
+              "end": 8,
+              "length": 1
+            }
+          ]
+        }
+      },
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          12
+        ],
+        "result": {
+          "index": 0,
+          "captureIndices": [
+            {
+              "start": 12,
+              "end": 12,
+              "length": 0
+            },
+            {
+              "start": 4294967295,
+              "end": 4294967295,
+              "length": 0
+            }
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "constractor": [
+      [
+        "\"",
+        "\\\\(?:[\"\\\\/bfnrt]|u[0-9a-fA-F]{4})",
+        "\\\\."
+      ]
+    ],
+    "executions": [
+      {
+        "args": [
+          "{\"foo\":\"bar\"}\n",
+          8
+        ],
+        "result": {
+          "index": 0,
+          "captureIndices": [
+            {
+              "start": 11,
+              "end": 12,
+              "length": 1
+            }
+          ]
+        }
+      }
+    ]
+  }
+]
diff --git a/packages/shiki/test/engine-js/__records__/markdown.json b/packages/shiki/test/engine-js/__records__/markdown.json
diff --git a/packages/shiki/test/engine-js/__records__/sql.json b/packages/shiki/test/engine-js/__records__/sql.json
diff --git a/packages/shiki/test/engine-js/__records__/toml.json b/packages/shiki/test/engine-js/__records__/toml.json
diff --git a/packages/shiki/test/engine-js/__records__/vue.json b/packages/shiki/test/engine-js/__records__/vue.json
diff --git a/packages/shiki/test/engine-js/compare.test.ts b/packages/shiki/test/engine-js/compare.test.ts
@@ -37,6 +37,7 @@ function createWasmOnigLibWrapper(): RegexEngine & { instances: Instance[] } {
 
 export interface Cases {
   name: string
+  skip?: boolean
   theme: () => Promise<{ default: ThemeRegistration }>
   lang: () => Promise<{ default: LanguageRegistration[] }>
   cases: string[]
@@ -69,6 +70,78 @@ const cases: Cases[] = [
       'const foo: string = "bar"',
     ],
   },
+  {
+    name: 'jsonc',
+    theme: () => import('../../src/assets/themes/nord'),
+    lang: () => import('../../src/assets/langs/jsonc'),
+    cases: [
+      '// comment\n{"foo":"bar"}',
+    ],
+  },
+  {
+    name: 'vue',
+    theme: () => import('../../src/assets/themes/vitesse-dark'),
+    lang: () => import('../../src/assets/langs/vue'),
+    cases: [
+      `<script setup>\nimport { ref } from 'vue'\n</script>`,
+      `<template>\n<div>{{ foo }}</div>\n</template>`,
+    ],
+  },
+  {
+    name: 'toml',
+    theme: () => import('../../src/assets/themes/nord'),
+    lang: () => import('../../src/assets/langs/toml'),
+    cases: [
+      [
+        `# This is a TOML document`,
+        '',
+        `title = "TOML Example"`,
+        '',
+        '[owner]',
+        'name = "Tom Preston-Werner"',
+      ].join('\n'),
+    ],
+  },
+  {
+    name: 'sql',
+    theme: () => import('../../src/assets/themes/nord'),
+    lang: () => import('../../src/assets/langs/sql'),
+    cases: [
+      'SELECT * FROM foo',
+      [
+        'USE AdventureWorks2022;',
+        'GO',
+        'IF OBJECT_ID(\'dbo.NewProducts\', \'U\') IS NOT NULL',
+        'DROP TABLE dbo.NewProducts;',
+      ].join('\n'),
+    ],
+  },
+  {
+    skip: true,
+    name: 'markdown',
+    theme: () => import('../../src/assets/themes/nord'),
+    lang: () => import('../../src/assets/langs/markdown'),
+    cases: [
+      [
+        '# Header',
+        '',
+        'This is a paragraph',
+        '',
+        '```ts',
+        'const foo = "bar"',
+        '```',
+      ].join('\n'),
+      [
+        'look like:',
+        '',
+        '  * this one',
+        '  * that one',
+        '  * the other one',
+        '',
+        'and this',
+      ].join('\n'),
+    ],
+  },
 ]
 
 describe('cases', async () => {
@@ -83,19 +156,20 @@ describe('cases', async () => {
   }))
 
   for (const c of resolved) {
-    it(c.c.name, async () => {
-      const wasm = createWasmOnigLibWrapper()
-      const native = createJavaScriptRegexEngine()
+    const run = c.c.skip ? it.skip : it
+    run(c.c.name, async () => {
+      const engineWasm = createWasmOnigLibWrapper()
+      const engineJs = createJavaScriptRegexEngine()
 
       const shiki1 = await createHighlighterCore({
         langs: c.lang,
         themes: [c.theme],
-        engine: wasm,
+        engine: engineWasm,
       })
       const shiki2 = await createHighlighterCore({
         langs: c.lang,
         themes: [c.theme],
-        engine: native,
+        engine: engineJs,
       })
 
       const lang = c.lang[0].name
@@ -110,12 +184,14 @@ describe('cases', async () => {
         ])
       }
 
-      await expect(JSON.stringify(wasm.instances, null, 2))
+      await expect.soft(JSON.stringify(engineWasm.instances, null, 2))
         .toMatchFileSnapshot(`./__records__/${c.c.name}.json`)
 
-      for (const [a, b] of compare) {
+      compare.forEach(([a, b]) => {
         expect.soft(a).toEqual(b)
-      }
+        // await expect.soft(a)
+        //   .toMatchFileSnapshot(`./__records__/tokens/${c.c.name}-${i}.json`)
+      })
     })
   }
 })
diff --git a/packages/shiki/test/engine-js/general.test.ts b/packages/shiki/test/engine-js/general.test.ts
@@ -42,6 +42,6 @@ describe('should', () => {
       `)
 
     expect(shiki.codeToHtml('@media foo { .bar { padding-left: 1px; }}', { lang: 'css', theme: 'min-dark' }))
-      .toMatchInlineSnapshot(`"<pre class="shiki min-dark" style="background-color:#1f1f1f;color:#b392f0" tabindex="0"><code><span class="line"><span style="color:#B392F0">@media foo { .bar { </span><span style="color:#79B8FF">padding-left</span><span style="color:#F97583">:</span><span style="color:#F8F8F8"> 1px</span><span style="color:#B392F0">; }}</span></span></code></pre>"`)
+      .toMatchInlineSnapshot(`"<pre class="shiki min-dark" style="background-color:#1f1f1f;color:#b392f0" tabindex="0"><code><span class="line"><span style="color:#F97583">@media</span><span style="color:#B392F0"> foo { .bar { </span><span style="color:#79B8FF">padding-left</span><span style="color:#F97583">:</span><span style="color:#F8F8F8"> 1px</span><span style="color:#B392F0">; }}</span></span></code></pre>"`)
   })
 })
diff --git a/packages/shiki/test/engine-js/verify.test.ts b/packages/shiki/test/engine-js/verify.test.ts
@@ -17,23 +17,36 @@ const cache = new Map<string, RegExp | Error>()
 for (const file of files) {
   // Some token positions are off in this record
   const name = basename(file, '.json')
-  if (name === 'ts-basic')
+
+  // TODO: markdown support is still problematic
+  if (name === 'markdown')
     continue
 
   describe(`record: ${name}`, async () => {
     const instances = JSON.parse(await fs.readFile(file, 'utf-8')) as Instance[]
     let i = 0
     for (const instance of instances) {
-      describe(`instances ${i++}`, () => {
+      i += 1
+      describe(`instances ${i}`, () => {
         const scanner = new JavaScriptScanner(instance.constractor[0], cache, false)
         let j = 0
         for (const execution of instance.executions) {
-          it(`case ${j++}`, () => {
+          j += 1
+          it(`case ${j}`, () => {
             onTestFailed(() => {
-              console.error({
-                patterns: scanner.patterns,
-                regexps: scanner.regexps,
-              })
+              console.error(execution.result?.index != null
+                ? {
+                    args: execution.args,
+                    expected: {
+                      pattern: scanner.patterns[execution.result.index],
+                      regexp: scanner.regexps[execution.result.index],
+                    },
+                  }
+                : {
+                    args: execution.args,
+                    patterns: scanner.patterns,
+                    regexps: scanner.regexps,
+                  })
             })
             const result = scanner.findNextMatchSync(...execution.args)
             expect(result).toEqual(execution.result)

diff --git a/packages/transformers/package.json b/packages/transformers/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/transformers",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "Collective of common transformers transformers for Shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/packages/twoslash/package.json b/packages/twoslash/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/twoslash",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "Shiki transformer for twoslash",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/packages/vitepress-twoslash/package.json b/packages/vitepress-twoslash/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/vitepress-twoslash",
   "type": "module",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "Enable Twoslash support in VitePress",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
@@ -41,6 +41,7 @@ catalog:
   hast-util-from-html: ^2.0.2
   hast-util-to-html: ^9.0.2
   hast-util-to-string: ^3.0.0
+  jiti: ^1.21.6
   jsonc-parser: ^3.3.1
   lint-staged: ^15.2.9
   markdown-it: ^14.1.0
@@ -50,7 +51,7 @@ catalog:
   minimist: ^1.2.8
   monaco-editor-core: ^0.51.0
   ofetch: ^1.3.4
-  oniguruma-to-js: ^0.2.3
+  oniguruma-to-js: ^0.2.4
   picocolors: ^1.0.1
   pinia: ^2.2.2
   pnpm: ^9.9.0

diff --git a/scripts/report-engine-js-compat.md b/scripts/report-engine-js-compat.md
diff --git a/scripts/report-engine-js-compat.ts b/scripts/report-engine-js-compat.ts
@@ -1,3 +1,7 @@
+// To run this script
+// - `pnpm run build`
+// - `pnpm run report-engine-js`
+
 import fs from 'node:fs/promises'
 import process from 'node:process'
 import { bundledLanguages, createHighlighter, createJavaScriptRegexEngine } from 'shiki'
@@ -18,6 +22,8 @@ export interface ReportItem {
 async function run() {
   const report: ReportItem[] = []
 
+  await fs.rm(new URL('./compares', import.meta.url), { recursive: true, force: true })
+
   for (const lang of Object.keys(bundledLanguages)) {
     const sample = await fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')
       .catch(() => '')
@@ -66,6 +72,25 @@ async function run() {
 
       if (!highlightMatch) {
         console.log(c.yellow(`[${lang}] Mismatch`))
+
+        await fs.mkdir(new URL('./compares', import.meta.url), { recursive: true })
+
+        await fs.writeFile(
+          new URL(`./compares/${lang}.html`, import.meta.url),
+          [
+            '<style>',
+            'body { display: flex; }',
+            'pre { flex: 1; margin: 0; padding: 0; }',
+            '</style>',
+            '<pre>',
+            highlightA,
+            '</pre>',
+            '<pre>',
+            highlightB,
+            '</pre>',
+          ].join('\n'),
+          'utf-8',
+        )
       }
       else {
         console.log(c.green(`[${lang}] OK`))