fix: support more languages

Author: antfu

packages/core/src/engines/javascript.ts

@@ -25,6 +25,8 @@ export class JavaScriptScanner implements PatternScanner {
       try {
         const regex = onigurumaToRegexp(
           p
+            .replace(/\|\\G(\||\))/g, '$1')
+            .replace(/(\(|\|)\\G\|/g, '$1')
             // YAML specific handling; TODO: move to tm-grammars
             .replaceAll('[^\\s[-?:,\\[\\]{}#&*!|>\'"%@`]]', '[^\\s\\-?:,\\[\\]{}#&*!|>\'"%@`]'),
           {

packages/shiki/test/engine-js/__records__/markdown.json

@@ -115,6 +115,31 @@
       ].join('\n'),
     ],
   },
+  {
+    name: 'markdown',
+    theme: () => import('../../src/assets/themes/nord'),
+    lang: () => import('../../src/assets/langs/markdown'),
+    cases: [
+      [
+        '# Header',
+        '',
+        'This is a paragraph',
+        '',
+        '```ts',
+        'const foo = "bar"',
+        '```',
+      ].join('\n'),
+      [
+        'look like:',
+        '',
+        '  * this one',
+        '  * that one',
+        '  * the other one',
+        '',
+        'and this',
+      ].join('\n'),
+    ],
+  },
 ]
 
 describe('cases', async () => {
@@ -160,7 +185,7 @@
         .toMatchFileSnapshot(`./__records__/${c.c.name}.json`)
 
       compare.forEach(([a, b]) => {
         expect.soft(a).toEqual(b)
         // await expect.soft(a)
         //   .toMatchFileSnapshot(`./__records__/tokens/${c.c.name}-${i}.json`)
       })

packages/shiki/test/engine-js/compare.test.ts

@@ -17,7 +17,9 @@ const cache = new Map<string, RegExp | Error>()
 for (const file of files) {
   // Some token positions are off in this record
   const name = basename(file, '.json')
-  if (name === 'ts-basic')
+
+  // TODO: markdown support is still problematic
+  if (name === 'markdown')
     continue
 
   describe(`record: ${name}`, async () => {

packages/shiki/test/engine-js/verify.test.ts

@@ -9,8 +9,8 @@
 | Status          | Number |
 | :-------------- | -----: |
 | Total Languages |    213 |
-| OK              |    178 |
-| Mismatch        |     33 |
+| OK              |    179 |
+| Mismatch        |     32 |
 | Error           |      2 |
 
 | Language           | Highlight Match | Patterns Parsable | Patterns Failed |
@@ -38,6 +38,7 @@
 | cmake              | ✅ OK           | 23                | -               |
 | cobol              | ✅ OK           | 138               | -               |
 | codeowners         | ✅ OK           | 4                 | -               |
+| codeql             | ✅ OK           | 150               | -               |
 | coffee             | ✅ OK           | 120               | -               |
 | common-lisp        | ✅ OK           | 57                | -               |
 | cpp                | ✅ OK           | 220               | -               |
@@ -199,7 +200,6 @@
 | beancount          | ⚠️ Mismatch     | 39                | -               |
 | blade              | ⚠️ Mismatch     | 330               | -               |
 | cadence            | ⚠️ Mismatch     | 71                | -               |
-| codeql             | ⚠️ Mismatch     | 150               | -               |
 | coq                | ⚠️ Mismatch     | 25                | -               |
 | cue                | ⚠️ Mismatch     | 85                | -               |
 | cypher             | ⚠️ Mismatch     | 39                | -               |