fix(shiki): issue with merging multiple themes (#1703)

nuxt · Nov 22, 2022 · 29f088c · 29f088c
1 parent 7c424d3
commit 29f088c
Show file tree

Hide file tree

Showing 2 changed files with 156 additions and 47 deletions.
diff --git a/src/runtime/server/api/highlight.ts b/src/runtime/server/api/highlight.ts
@@ -152,51 +152,54 @@ interface HighlightThemedTokenLine {
 
 function mergeLines (line1: HighlightThemedTokenLine, line2: HighlightThemedTokenLine) {
   const mergedTokens: HighlightThemedToken[] = []
-  const getColors = (h, i) => typeof h.tokens[i].color === 'string' ? { [h.key]: h.tokens[i].color } : h.tokens[i].color
-
-  const [big, small] = line1.tokens.length > line2.tokens.length ? [line1, line2] : [line2, line1]
-  let targetToken = 0
-  let targetTokenCharIndex = 0
-  big.tokens.forEach((t, i) => {
-    if (targetTokenCharIndex === 0) {
-      if (t.content === small.tokens[i]?.content) {
-        mergedTokens.push({
-          content: t.content,
-          color: {
-            ...getColors(big, i),
-            ...getColors(small, i)
-          }
-        })
-        targetToken = i + 1
-        return
-      }
-      if (t.content === small.tokens[targetToken]?.content) {
-        mergedTokens.push({
-          content: t.content,
-          color: {
-            ...getColors(big, i),
-            ...getColors(small, targetToken)
-          }
-        })
-        targetToken += 1
-        return
-      }
-    }
+  const getColors = (h: HighlightThemedTokenLine, i: number) => typeof h.tokens[i].color === 'string' ? { [h.key]: h.tokens[i].color } : h.tokens[i].color as object
+
+  const right = {
+    key: line1.key,
+    tokens: line1.tokens.slice()
+  }
+  const left = {
+    key: line2.key,
+    tokens: line2.tokens.slice()
+  }
+  let index = 0
+  while (index < right.tokens.length) {
+    const rightToken = right.tokens[index]
+    const leftToken = left.tokens[index]
 
-    if (small.tokens[targetToken]?.content?.substring(targetTokenCharIndex, targetTokenCharIndex + t.content.length) === t.content) {
-      targetTokenCharIndex += t.content.length
+    if (rightToken.content === leftToken.content) {
       mergedTokens.push({
-        content: t.content,
+        content: rightToken.content,
         color: {
-          ...getColors(big, i),
-          ...getColors(small, targetToken)
+          ...getColors(right, index),
+          ...getColors(left, index)
         }
       })
+      index += 1
+      continue
     }
-    if (small.tokens[targetToken]?.content.length <= targetTokenCharIndex) {
-      targetToken += 1
-      targetTokenCharIndex = 0
+
+    if (rightToken.content.startsWith(leftToken.content)) {
+      const nextRightToken = {
+        ...rightToken,
+        content: rightToken.content.slice(leftToken.content.length)
+      }
+      rightToken.content = leftToken.content
+      right.tokens.splice(index + 1, 0, nextRightToken)
+      continue
     }
-  })
+
+    if (leftToken.content.startsWith(rightToken.content)) {
+      const nextLeftToken = {
+        ...leftToken,
+        content: leftToken.content.slice(rightToken.content.length)
+      }
+      leftToken.content = rightToken.content
+      left.tokens.splice(index + 1, 0, nextLeftToken)
+      continue
+    }
+
+    throw new Error('Unexpected token')
+  }
   return mergedTokens
 }
diff --git a/test/features/highlighter.ts b/test/features/highlighter.ts
@@ -1,20 +1,21 @@
 import { describe, test, expect, assert } from 'vitest'
-import { $fetch } from '@nuxt/test-utils'
-
-const content = [
-  '```ts',
-  'const a: number = 1',
-  '```'
-].join('\n')
+import { $fetch, useTestContext } from '@nuxt/test-utils'
 
 export const testHighlighter = () => {
+  // @ts-ignore
+  const apiBaseUrl = useTestContext().options.nuxtConfig.content?.api?.baseURL || '/api/_content'
+
   describe('Highlighter', () => {
     test('themed', async () => {
       const parsed = await $fetch('/api/parse', {
         method: 'POST',
         body: {
           id: 'content:index.md',
-          content
+          content: [
+            '```ts',
+            'const a: number = 1',
+            '```'
+          ].join('\n')
         }
       })
 
@@ -57,5 +58,110 @@ export const testHighlighter = () => {
       expect(style).toContain(`.${code[9].props.class}{color:#0550AE}`)
       expect(style).toContain(`.dark .${code[9].props.class}{color:#79C0FF}`)
     })
+
+    test('highlight multi-theme with different tokenizer', async () => {
+      const tokens = await $fetch(`${apiBaseUrl}/highlight`, {
+        method: 'POST',
+        body: {
+          lang: 'ts',
+          theme: {
+            dark: 'one-dark-pro',
+            default: 'github-light'
+          },
+          code: 'type UseFetchOptions = { key?: string }'
+        }
+      })
+
+      expect(tokens).toMatchInlineSnapshot(`
+        [
+          [
+            {
+              "color": {
+                "dark": "#C678DD",
+                "default": "#CF222E",
+              },
+              "content": "type",
+            },
+            {
+              "color": {
+                "dark": "#ABB2BF",
+                "default": "#24292F",
+              },
+              "content": " ",
+            },
+            {
+              "color": {
+                "dark": "#E5C07B",
+                "default": "#953800",
+              },
+              "content": "UseFetchOptions",
+            },
+            {
+              "color": {
+                "dark": "#ABB2BF",
+                "default": "#24292F",
+              },
+              "content": " ",
+            },
+            {
+              "color": {
+                "dark": "#56B6C2",
+                "default": "#CF222E",
+              },
+              "content": "=",
+            },
+            {
+              "color": {
+                "dark": "#ABB2BF",
+                "default": "#24292F",
+              },
+              "content": " { ",
+            },
+            {
+              "color": {
+                "dark": "#E06C75",
+                "default": "#953800",
+              },
+              "content": "key",
+            },
+            {
+              "color": {
+                "dark": "#C678DD",
+                "default": "#CF222E",
+              },
+              "content": "?",
+            },
+            {
+              "color": {
+                "dark": "#ABB2BF",
+                "default": "#CF222E",
+              },
+              "content": ":",
+            },
+            {
+              "color": {
+                "dark": "#ABB2BF",
+                "default": "#24292F",
+              },
+              "content": " ",
+            },
+            {
+              "color": {
+                "dark": "#E5C07B",
+                "default": "#0550AE",
+              },
+              "content": "string",
+            },
+            {
+              "color": {
+                "dark": "#ABB2BF",
+                "default": "#24292F",
+              },
+              "content": " }",
+            },
+          ],
+        ]
+      `)
+    })
   })
 }