Skip to content

Commit efbb7fb

Browse files
antfuantfu
authoredAug 14, 2024··
feat(core): lighter token explanation with scopeName (#739)
·
v3.2.2v1.13.0
1 parent 933415c commit efbb7fb

File tree

4 files changed

+698
-28
lines changed

4 files changed

+698
-28
lines changed
 

‎packages/core/src/code-to-tokens-base.ts

+39-26
Original file line numberDiff line numberDiff line change
@@ -125,28 +125,6 @@ function _tokenizeWithTheme(
125125
let actual: ThemedToken[] = []
126126
const final: ThemedToken[][] = []
127127

128-
const themeSettingsSelectors: ThemeSettingsSelectors[] = []
129-
if (options.includeExplanation) {
130-
for (const setting of theme.settings) {
131-
let selectors: string[]
132-
switch (typeof setting.scope) {
133-
case 'string':
134-
selectors = setting.scope.split(/,/).map(scope => scope.trim())
135-
break
136-
case 'object':
137-
selectors = setting.scope
138-
break
139-
default:
140-
continue
141-
}
142-
143-
themeSettingsSelectors.push({
144-
settings: setting,
145-
selectors: selectors.map(selector => selector.split(/ /)),
146-
})
147-
}
148-
}
149-
150128
for (let i = 0, len = lines.length; i < len; i++) {
151129
const [line, lineOffset] = lines[i]
152130
if (line === '') {
@@ -201,6 +179,29 @@ function _tokenizeWithTheme(
201179
}
202180

203181
if (options.includeExplanation) {
182+
const themeSettingsSelectors: ThemeSettingsSelectors[] = []
183+
184+
if (options.includeExplanation !== 'scopeName') {
185+
for (const setting of theme.settings) {
186+
let selectors: string[]
187+
switch (typeof setting.scope) {
188+
case 'string':
189+
selectors = setting.scope.split(/,/).map(scope => scope.trim())
190+
break
191+
case 'object':
192+
selectors = setting.scope
193+
break
194+
default:
195+
continue
196+
}
197+
198+
themeSettingsSelectors.push({
199+
settings: setting,
200+
selectors: selectors.map(selector => selector.split(/ /)),
201+
})
202+
}
203+
}
204+
204205
token.explanation = []
205206
let offset = 0
206207
while (startIndex + offset < nextStartIndex) {
@@ -213,7 +214,14 @@ function _tokenizeWithTheme(
213214
offset += tokenWithScopesText.length
214215
token.explanation.push({
215216
content: tokenWithScopesText,
216-
scopes: explainThemeScopes(themeSettingsSelectors, tokenWithScopes.scopes),
217+
scopes: options.includeExplanation === 'scopeName'
218+
? explainThemeScopesNameOnly(
219+
tokenWithScopes.scopes,
220+
)
221+
: explainThemeScopesFull(
222+
themeSettingsSelectors,
223+
tokenWithScopes.scopes,
224+
),
217225
})
218226

219227
tokensWithScopesIndex! += 1
@@ -233,17 +241,22 @@ function _tokenizeWithTheme(
233241
}
234242
}
235243

236-
function explainThemeScopes(
244+
function explainThemeScopesNameOnly(
245+
scopes: string[],
246+
): ThemedTokenScopeExplanation[] {
247+
return scopes.map(scope => ({ scopeName: scope }))
248+
}
249+
250+
function explainThemeScopesFull(
237251
themeSelectors: ThemeSettingsSelectors[],
238252
scopes: string[],
239253
): ThemedTokenScopeExplanation[] {
240254
const result: ThemedTokenScopeExplanation[] = []
241255
for (let i = 0, len = scopes.length; i < len; i++) {
242-
const parentScopes = scopes.slice(0, i)
243256
const scope = scopes[i]
244257
result[i] = {
245258
scopeName: scope,
246-
themeMatches: explainThemeScope(themeSelectors, scope, parentScopes),
259+
themeMatches: explainThemeScope(themeSelectors, scope, scopes.slice(0, i)),
247260
}
248261
}
249262
return result

‎packages/core/src/types/tokens.ts

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { GrammarState } from '../grammar-state'
2+
import type { IRawThemeSetting } from '../textmate'
23
import type { SpecialLanguage } from './langs'
34
import type { SpecialTheme, ThemeRegistrationAny } from './themes'
45
import type { CodeOptionsThemes } from './options'
@@ -36,7 +37,7 @@ export interface CodeToTokensWithThemesOptions<Languages = string, Themes = stri
3637

3738
export interface ThemedTokenScopeExplanation {
3839
scopeName: string
39-
themeMatches: any[]
40+
themeMatches?: IRawThemeSetting[]
4041
}
4142

4243
export interface ThemedTokenExplanation {
@@ -149,9 +150,12 @@ export interface TokenizeWithThemeOptions {
149150
/**
150151
* Include explanation of why a token is given a color.
151152
*
153+
* You can optionally pass `scopeName` to only include explanation for scopes,
154+
* which is more performant than full explanation.
155+
*
152156
* @default false
153157
*/
154-
includeExplanation?: boolean
158+
includeExplanation?: boolean | 'scopeName'
155159

156160
/**
157161
* A map of color names to new color values.

‎packages/shiki/test/__snapshots__/tokens.test.ts.snap

+640
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,640 @@
1+
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2+
3+
exports[`includeExplanation > false 1`] = `
4+
[
5+
[
6+
{
7+
"color": "#BD976A",
8+
"content": "console",
9+
"fontStyle": 0,
10+
"offset": 0,
11+
},
12+
{
13+
"color": "#666666",
14+
"content": ".",
15+
"fontStyle": 0,
16+
"offset": 7,
17+
},
18+
{
19+
"color": "#80A665",
20+
"content": "log",
21+
"fontStyle": 0,
22+
"offset": 8,
23+
},
24+
{
25+
"color": "#666666",
26+
"content": "(",
27+
"fontStyle": 0,
28+
"offset": 11,
29+
},
30+
{
31+
"color": "#C98A7D77",
32+
"content": """,
33+
"fontStyle": 0,
34+
"offset": 12,
35+
},
36+
{
37+
"color": "#C98A7D",
38+
"content": "hello",
39+
"fontStyle": 0,
40+
"offset": 13,
41+
},
42+
{
43+
"color": "#C98A7D77",
44+
"content": """,
45+
"fontStyle": 0,
46+
"offset": 18,
47+
},
48+
{
49+
"color": "#666666",
50+
"content": ")",
51+
"fontStyle": 0,
52+
"offset": 19,
53+
},
54+
],
55+
]
56+
`;
57+
58+
exports[`includeExplanation > scopeName 1`] = `
59+
[
60+
[
61+
{
62+
"color": "#BD976A",
63+
"content": "console",
64+
"explanation": [
65+
{
66+
"content": "console",
67+
"scopes": [
68+
{
69+
"scopeName": "source.js",
70+
},
71+
{
72+
"scopeName": "meta.function-call.js",
73+
},
74+
{
75+
"scopeName": "variable.other.object.js",
76+
},
77+
],
78+
},
79+
],
80+
"fontStyle": 0,
81+
"offset": 0,
82+
},
83+
{
84+
"color": "#666666",
85+
"content": ".",
86+
"explanation": [
87+
{
88+
"content": ".",
89+
"scopes": [
90+
{
91+
"scopeName": "source.js",
92+
},
93+
{
94+
"scopeName": "meta.function-call.js",
95+
},
96+
{
97+
"scopeName": "punctuation.accessor.js",
98+
},
99+
],
100+
},
101+
],
102+
"fontStyle": 0,
103+
"offset": 7,
104+
},
105+
{
106+
"color": "#80A665",
107+
"content": "log",
108+
"explanation": [
109+
{
110+
"content": "log",
111+
"scopes": [
112+
{
113+
"scopeName": "source.js",
114+
},
115+
{
116+
"scopeName": "meta.function-call.js",
117+
},
118+
{
119+
"scopeName": "entity.name.function.js",
120+
},
121+
],
122+
},
123+
],
124+
"fontStyle": 0,
125+
"offset": 8,
126+
},
127+
{
128+
"color": "#666666",
129+
"content": "(",
130+
"explanation": [
131+
{
132+
"content": "(",
133+
"scopes": [
134+
{
135+
"scopeName": "source.js",
136+
},
137+
{
138+
"scopeName": "meta.brace.round.js",
139+
},
140+
],
141+
},
142+
],
143+
"fontStyle": 0,
144+
"offset": 11,
145+
},
146+
{
147+
"color": "#C98A7D77",
148+
"content": """,
149+
"explanation": [
150+
{
151+
"content": """,
152+
"scopes": [
153+
{
154+
"scopeName": "source.js",
155+
},
156+
{
157+
"scopeName": "string.quoted.double.js",
158+
},
159+
{
160+
"scopeName": "punctuation.definition.string.begin.js",
161+
},
162+
],
163+
},
164+
],
165+
"fontStyle": 0,
166+
"offset": 12,
167+
},
168+
{
169+
"color": "#C98A7D",
170+
"content": "hello",
171+
"explanation": [
172+
{
173+
"content": "hello",
174+
"scopes": [
175+
{
176+
"scopeName": "source.js",
177+
},
178+
{
179+
"scopeName": "string.quoted.double.js",
180+
},
181+
],
182+
},
183+
],
184+
"fontStyle": 0,
185+
"offset": 13,
186+
},
187+
{
188+
"color": "#C98A7D77",
189+
"content": """,
190+
"explanation": [
191+
{
192+
"content": """,
193+
"scopes": [
194+
{
195+
"scopeName": "source.js",
196+
},
197+
{
198+
"scopeName": "string.quoted.double.js",
199+
},
200+
{
201+
"scopeName": "punctuation.definition.string.end.js",
202+
},
203+
],
204+
},
205+
],
206+
"fontStyle": 0,
207+
"offset": 18,
208+
},
209+
{
210+
"color": "#666666",
211+
"content": ")",
212+
"explanation": [
213+
{
214+
"content": ")",
215+
"scopes": [
216+
{
217+
"scopeName": "source.js",
218+
},
219+
{
220+
"scopeName": "meta.brace.round.js",
221+
},
222+
],
223+
},
224+
],
225+
"fontStyle": 0,
226+
"offset": 19,
227+
},
228+
],
229+
]
230+
`;
231+
232+
exports[`includeExplanation > true 1`] = `
233+
[
234+
[
235+
{
236+
"color": "#BD976A",
237+
"content": "console",
238+
"explanation": [
239+
{
240+
"content": "console",
241+
"scopes": [
242+
{
243+
"scopeName": "source.js",
244+
"themeMatches": [],
245+
},
246+
{
247+
"scopeName": "meta.function-call.js",
248+
"themeMatches": [],
249+
},
250+
{
251+
"scopeName": "variable.other.object.js",
252+
"themeMatches": [
253+
{
254+
"scope": [
255+
"variable",
256+
"identifier",
257+
],
258+
"settings": {
259+
"foreground": "#bd976a",
260+
},
261+
},
262+
],
263+
},
264+
],
265+
},
266+
],
267+
"fontStyle": 0,
268+
"offset": 0,
269+
},
270+
{
271+
"color": "#666666",
272+
"content": ".",
273+
"explanation": [
274+
{
275+
"content": ".",
276+
"scopes": [
277+
{
278+
"scopeName": "source.js",
279+
"themeMatches": [],
280+
},
281+
{
282+
"scopeName": "meta.function-call.js",
283+
"themeMatches": [],
284+
},
285+
{
286+
"scopeName": "punctuation.accessor.js",
287+
"themeMatches": [
288+
{
289+
"scope": [
290+
"delimiter.bracket",
291+
"delimiter",
292+
"invalid.illegal.character-not-allowed-here.html",
293+
"keyword.operator.rest",
294+
"keyword.operator.spread",
295+
"keyword.operator.type.annotation",
296+
"keyword.operator.relational",
297+
"keyword.operator.assignment",
298+
"keyword.operator.type",
299+
"meta.brace",
300+
"meta.tag.block.any.html",
301+
"meta.tag.inline.any.html",
302+
"meta.tag.structure.input.void.html",
303+
"meta.type.annotation",
304+
"meta.embedded.block.github-actions-expression",
305+
"storage.type.function.arrow",
306+
"meta.objectliteral.ts",
307+
"punctuation",
308+
"punctuation.definition.string.begin.html.vue",
309+
"punctuation.definition.string.end.html.vue",
310+
],
311+
"settings": {
312+
"foreground": "#666666",
313+
},
314+
},
315+
],
316+
},
317+
],
318+
},
319+
],
320+
"fontStyle": 0,
321+
"offset": 7,
322+
},
323+
{
324+
"color": "#80A665",
325+
"content": "log",
326+
"explanation": [
327+
{
328+
"content": "log",
329+
"scopes": [
330+
{
331+
"scopeName": "source.js",
332+
"themeMatches": [],
333+
},
334+
{
335+
"scopeName": "meta.function-call.js",
336+
"themeMatches": [],
337+
},
338+
{
339+
"scopeName": "entity.name.function.js",
340+
"themeMatches": [
341+
{
342+
"scope": [
343+
"entity",
344+
"entity.name",
345+
],
346+
"settings": {
347+
"foreground": "#80a665",
348+
},
349+
},
350+
{
351+
"scope": "entity.name.function",
352+
"settings": {
353+
"foreground": "#80a665",
354+
},
355+
},
356+
],
357+
},
358+
],
359+
},
360+
],
361+
"fontStyle": 0,
362+
"offset": 8,
363+
},
364+
{
365+
"color": "#666666",
366+
"content": "(",
367+
"explanation": [
368+
{
369+
"content": "(",
370+
"scopes": [
371+
{
372+
"scopeName": "source.js",
373+
"themeMatches": [],
374+
},
375+
{
376+
"scopeName": "meta.brace.round.js",
377+
"themeMatches": [
378+
{
379+
"scope": [
380+
"delimiter.bracket",
381+
"delimiter",
382+
"invalid.illegal.character-not-allowed-here.html",
383+
"keyword.operator.rest",
384+
"keyword.operator.spread",
385+
"keyword.operator.type.annotation",
386+
"keyword.operator.relational",
387+
"keyword.operator.assignment",
388+
"keyword.operator.type",
389+
"meta.brace",
390+
"meta.tag.block.any.html",
391+
"meta.tag.inline.any.html",
392+
"meta.tag.structure.input.void.html",
393+
"meta.type.annotation",
394+
"meta.embedded.block.github-actions-expression",
395+
"storage.type.function.arrow",
396+
"meta.objectliteral.ts",
397+
"punctuation",
398+
"punctuation.definition.string.begin.html.vue",
399+
"punctuation.definition.string.end.html.vue",
400+
],
401+
"settings": {
402+
"foreground": "#666666",
403+
},
404+
},
405+
],
406+
},
407+
],
408+
},
409+
],
410+
"fontStyle": 0,
411+
"offset": 11,
412+
},
413+
{
414+
"color": "#C98A7D77",
415+
"content": """,
416+
"explanation": [
417+
{
418+
"content": """,
419+
"scopes": [
420+
{
421+
"scopeName": "source.js",
422+
"themeMatches": [],
423+
},
424+
{
425+
"scopeName": "string.quoted.double.js",
426+
"themeMatches": [
427+
{
428+
"scope": [
429+
"string",
430+
"string punctuation.section.embedded source",
431+
"attribute.value",
432+
],
433+
"settings": {
434+
"foreground": "#c98a7d",
435+
},
436+
},
437+
],
438+
},
439+
{
440+
"scopeName": "punctuation.definition.string.begin.js",
441+
"themeMatches": [
442+
{
443+
"scope": [
444+
"delimiter.bracket",
445+
"delimiter",
446+
"invalid.illegal.character-not-allowed-here.html",
447+
"keyword.operator.rest",
448+
"keyword.operator.spread",
449+
"keyword.operator.type.annotation",
450+
"keyword.operator.relational",
451+
"keyword.operator.assignment",
452+
"keyword.operator.type",
453+
"meta.brace",
454+
"meta.tag.block.any.html",
455+
"meta.tag.inline.any.html",
456+
"meta.tag.structure.input.void.html",
457+
"meta.type.annotation",
458+
"meta.embedded.block.github-actions-expression",
459+
"storage.type.function.arrow",
460+
"meta.objectliteral.ts",
461+
"punctuation",
462+
"punctuation.definition.string.begin.html.vue",
463+
"punctuation.definition.string.end.html.vue",
464+
],
465+
"settings": {
466+
"foreground": "#666666",
467+
},
468+
},
469+
{
470+
"scope": [
471+
"punctuation.definition.string",
472+
],
473+
"settings": {
474+
"foreground": "#c98a7d77",
475+
},
476+
},
477+
],
478+
},
479+
],
480+
},
481+
],
482+
"fontStyle": 0,
483+
"offset": 12,
484+
},
485+
{
486+
"color": "#C98A7D",
487+
"content": "hello",
488+
"explanation": [
489+
{
490+
"content": "hello",
491+
"scopes": [
492+
{
493+
"scopeName": "source.js",
494+
"themeMatches": [],
495+
},
496+
{
497+
"scopeName": "string.quoted.double.js",
498+
"themeMatches": [
499+
{
500+
"scope": [
501+
"string",
502+
"string punctuation.section.embedded source",
503+
"attribute.value",
504+
],
505+
"settings": {
506+
"foreground": "#c98a7d",
507+
},
508+
},
509+
],
510+
},
511+
],
512+
},
513+
],
514+
"fontStyle": 0,
515+
"offset": 13,
516+
},
517+
{
518+
"color": "#C98A7D77",
519+
"content": """,
520+
"explanation": [
521+
{
522+
"content": """,
523+
"scopes": [
524+
{
525+
"scopeName": "source.js",
526+
"themeMatches": [],
527+
},
528+
{
529+
"scopeName": "string.quoted.double.js",
530+
"themeMatches": [
531+
{
532+
"scope": [
533+
"string",
534+
"string punctuation.section.embedded source",
535+
"attribute.value",
536+
],
537+
"settings": {
538+
"foreground": "#c98a7d",
539+
},
540+
},
541+
],
542+
},
543+
{
544+
"scopeName": "punctuation.definition.string.end.js",
545+
"themeMatches": [
546+
{
547+
"scope": [
548+
"delimiter.bracket",
549+
"delimiter",
550+
"invalid.illegal.character-not-allowed-here.html",
551+
"keyword.operator.rest",
552+
"keyword.operator.spread",
553+
"keyword.operator.type.annotation",
554+
"keyword.operator.relational",
555+
"keyword.operator.assignment",
556+
"keyword.operator.type",
557+
"meta.brace",
558+
"meta.tag.block.any.html",
559+
"meta.tag.inline.any.html",
560+
"meta.tag.structure.input.void.html",
561+
"meta.type.annotation",
562+
"meta.embedded.block.github-actions-expression",
563+
"storage.type.function.arrow",
564+
"meta.objectliteral.ts",
565+
"punctuation",
566+
"punctuation.definition.string.begin.html.vue",
567+
"punctuation.definition.string.end.html.vue",
568+
],
569+
"settings": {
570+
"foreground": "#666666",
571+
},
572+
},
573+
{
574+
"scope": [
575+
"punctuation.definition.string",
576+
],
577+
"settings": {
578+
"foreground": "#c98a7d77",
579+
},
580+
},
581+
],
582+
},
583+
],
584+
},
585+
],
586+
"fontStyle": 0,
587+
"offset": 18,
588+
},
589+
{
590+
"color": "#666666",
591+
"content": ")",
592+
"explanation": [
593+
{
594+
"content": ")",
595+
"scopes": [
596+
{
597+
"scopeName": "source.js",
598+
"themeMatches": [],
599+
},
600+
{
601+
"scopeName": "meta.brace.round.js",
602+
"themeMatches": [
603+
{
604+
"scope": [
605+
"delimiter.bracket",
606+
"delimiter",
607+
"invalid.illegal.character-not-allowed-here.html",
608+
"keyword.operator.rest",
609+
"keyword.operator.spread",
610+
"keyword.operator.type.annotation",
611+
"keyword.operator.relational",
612+
"keyword.operator.assignment",
613+
"keyword.operator.type",
614+
"meta.brace",
615+
"meta.tag.block.any.html",
616+
"meta.tag.inline.any.html",
617+
"meta.tag.structure.input.void.html",
618+
"meta.type.annotation",
619+
"meta.embedded.block.github-actions-expression",
620+
"storage.type.function.arrow",
621+
"meta.objectliteral.ts",
622+
"punctuation",
623+
"punctuation.definition.string.begin.html.vue",
624+
"punctuation.definition.string.end.html.vue",
625+
],
626+
"settings": {
627+
"foreground": "#666666",
628+
},
629+
},
630+
],
631+
},
632+
],
633+
},
634+
],
635+
"fontStyle": 0,
636+
"offset": 19,
637+
},
638+
],
639+
]
640+
`;

‎packages/shiki/test/tokens.test.ts

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { expect, it } from 'vitest'
2+
import { codeToTokensBase } from '../src'
3+
4+
it('includeExplanation', async () => {
5+
const code = 'console.log("hello")'
6+
const caseFalse = await codeToTokensBase(code, { lang: 'js', theme: 'vitesse-dark', includeExplanation: false })
7+
const caseTrue = await codeToTokensBase(code, { lang: 'js', theme: 'vitesse-dark', includeExplanation: true })
8+
const caseScopeName = await codeToTokensBase(code, { lang: 'js', theme: 'vitesse-dark', includeExplanation: 'scopeName' })
9+
10+
expect(caseFalse).toMatchSnapshot('false')
11+
expect(caseTrue).toMatchSnapshot('true')
12+
expect(caseScopeName).toMatchSnapshot('scopeName')
13+
})

0 commit comments

Comments
 (0)
Please sign in to comment.