Skip to content

Commit 9396a6f

Browse files
antfuantfu
authoredSep 1, 2024··
feat: synchronous Shiki usage (#764)
·
v3.2.1v1.16.0
1 parent eb842a3 commit 9396a6f

File tree

14 files changed

+337
-181
lines changed

14 files changed

+337
-181
lines changed
 

‎bench/engines.bench.ts

+30-26
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,43 @@
1+
/* eslint-disable no-console */
12
import fs from 'node:fs/promises'
23
import { bench, describe } from 'vitest'
34
import type { BundledLanguage } from 'shiki'
45
import { createHighlighter, createJavaScriptRegexEngine, createWasmOnigEngine } from 'shiki'
56
import type { ReportItem } from '../scripts/report-engine-js-compat'
67

7-
describe('engines', async () => {
8-
const js = createJavaScriptRegexEngine()
9-
const wasm = await createWasmOnigEngine(() => import('shiki/wasm'))
8+
const js = createJavaScriptRegexEngine()
9+
const wasm = await createWasmOnigEngine(() => import('shiki/wasm'))
1010

11-
// Run `npx jiti scripts/report-engine-js-compat.ts` to generate the report first
12-
const report = await fs.readFile('../scripts/report-engine-js-compat.json', 'utf-8').then(JSON.parse) as ReportItem[]
13-
const langs = report.filter(i => i.highlightMatch === true).map(i => i.lang) as BundledLanguage[]
14-
const samples = await Promise.all(langs.map(lang => fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')))
11+
const RANGE = [0, 20]
1512

16-
const shikiJs = await createHighlighter({
17-
langs,
18-
themes: ['vitesse-dark'],
19-
engine: js,
20-
})
13+
// Run `npx jiti scripts/report-engine-js-compat.ts` to generate the report first
14+
const report = await fs.readFile(new URL('../scripts/report-engine-js-compat.json', import.meta.url), 'utf-8').then(JSON.parse) as ReportItem[]
15+
const langs = report.filter(i => i.highlightMatch === true).map(i => i.lang).slice(...RANGE) as BundledLanguage[]
16+
// Clone https://github.com/shikijs/textmate-grammars-themes to `../tm-grammars-themes`
17+
const samples = await Promise.all(langs.map(lang => fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')))
2118

22-
const shikiWasm = await createHighlighter({
23-
langs,
24-
themes: ['vitesse-dark'],
25-
engine: wasm,
26-
})
19+
console.log('Benchmarking engines with', langs.length, 'languages')
20+
21+
const shikiJs = await createHighlighter({
22+
langs,
23+
themes: ['vitesse-dark'],
24+
engine: js,
25+
})
2726

28-
bench('js', () => {
29-
for (const lang of langs) {
27+
const shikiWasm = await createHighlighter({
28+
langs,
29+
themes: ['vitesse-dark'],
30+
engine: wasm,
31+
})
32+
33+
for (const lang of langs) {
34+
describe(lang, () => {
35+
bench('js', () => {
3036
shikiJs.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
31-
}
32-
}, { warmupIterations: 10, iterations: 30 })
37+
})
3338

34-
bench('wasm', () => {
35-
for (const lang of langs) {
39+
bench('wasm', () => {
3640
shikiWasm.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
37-
}
38-
}, { warmupIterations: 10, iterations: 30 })
39-
})
41+
})
42+
})
43+
}

‎packages/core/src/constructors/highlighter.ts

+24
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { codeToTokensBase, getLastGrammarState } from '../highlight/code-to-toke
55
import { codeToTokensWithThemes } from '../highlight/code-to-tokens-themes'
66
import type { HighlighterCore, HighlighterCoreOptions } from '../types'
77
import { createShikiInternal } from './internal'
8+
import { createShikiInternalSync } from './internal-sync'
89

910
/**
1011
* Create a Shiki core highlighter instance, with no languages or themes bundled.
@@ -27,6 +28,29 @@ export async function createHighlighterCore(options: HighlighterCoreOptions = {}
2728
}
2829
}
2930

31+
/**
32+
* Create a Shiki core highlighter instance, with no languages or themes bundled.
33+
* Wasm and each language and theme must be loaded manually.
34+
*
35+
* Synchronous version of `createHighlighterCore`, which requires to provide the engine and all themes and languages upfront.
36+
*
37+
* @see http://shiki.style/guide/install#fine-grained-bundle
38+
*/
39+
export function createHighlighterCoreSync(options: HighlighterCoreOptions<true> = {}): HighlighterCore {
40+
const internal = createShikiInternalSync(options)
41+
42+
return {
43+
getLastGrammarState: (code, options) => getLastGrammarState(internal, code, options),
44+
codeToTokensBase: (code, options) => codeToTokensBase(internal, code, options),
45+
codeToTokensWithThemes: (code, options) => codeToTokensWithThemes(internal, code, options),
46+
codeToTokens: (code, options) => codeToTokens(internal, code, options),
47+
codeToHast: (code, options) => codeToHast(internal, code, options),
48+
codeToHtml: (code, options) => codeToHtml(internal, code, options),
49+
...internal,
50+
getInternalContext: () => internal,
51+
}
52+
}
53+
3054
export function makeSingletonHighlighterCore(createHighlighter: typeof createHighlighterCore) {
3155
let _shiki: ReturnType<typeof createHighlighterCore>
3256

‎packages/core/src/constructors/internal-sync.ts

+133
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import type {
2+
HighlighterCoreOptions,
3+
LanguageInput,
4+
LanguageRegistration,
5+
MaybeArray,
6+
ShikiInternal,
7+
SpecialLanguage,
8+
SpecialTheme,
9+
ThemeInput,
10+
ThemeRegistrationAny,
11+
ThemeRegistrationResolved,
12+
} from '../types'
13+
import { Registry } from '../textmate/registry'
14+
import { Resolver } from '../textmate/resolver'
15+
import { normalizeTheme } from '../textmate/normalize-theme'
16+
import { ShikiError } from '../error'
17+
import { resolveLangs, resolveThemes } from '../textmate/getters-resolve'
18+
19+
let instancesCount = 0
20+
21+
/**
22+
* Get the minimal shiki context for rendering.
23+
*
24+
* Synchronous version of `createShikiInternal`, which requires to provide the engine and all themes and languages upfront.
25+
*/
26+
export function createShikiInternalSync(options: HighlighterCoreOptions<true>): ShikiInternal {
27+
instancesCount += 1
28+
if (options.warnings !== false && instancesCount >= 10 && instancesCount % 10 === 0)
29+
console.warn(`[Shiki] ${instancesCount} instances have been created. Shiki is supposed to be used as a singleton, consider refactoring your code to cache your highlighter instance; Or call \`highlighter.dispose()\` to release unused instances.`)
30+
31+
let isDisposed = false
32+
33+
if (!options.engine)
34+
throw new ShikiError('`engine` option is required for synchronous mode')
35+
36+
const langs = (options.langs || []).flat(1)
37+
const themes = (options.themes || []).flat(1).map(normalizeTheme)
38+
39+
const resolver = new Resolver(options.engine, langs)
40+
const _registry = new Registry(resolver, themes, langs, options.langAlias)
41+
42+
let _lastTheme: string | ThemeRegistrationAny
43+
44+
function getLanguage(name: string | LanguageRegistration) {
45+
ensureNotDisposed()
46+
const _lang = _registry.getGrammar(typeof name === 'string' ? name : name.name)
47+
if (!_lang)
48+
throw new ShikiError(`Language \`${name}\` not found, you may need to load it first`)
49+
return _lang
50+
}
51+
52+
function getTheme(name: string | ThemeRegistrationAny): ThemeRegistrationResolved {
53+
if (name === 'none')
54+
return { bg: '', fg: '', name: 'none', settings: [], type: 'dark' }
55+
ensureNotDisposed()
56+
const _theme = _registry.getTheme(name)
57+
if (!_theme)
58+
throw new ShikiError(`Theme \`${name}\` not found, you may need to load it first`)
59+
return _theme
60+
}
61+
62+
function setTheme(name: string | ThemeRegistrationAny) {
63+
ensureNotDisposed()
64+
const theme = getTheme(name)
65+
if (_lastTheme !== name) {
66+
_registry.setTheme(theme)
67+
_lastTheme = name
68+
}
69+
const colorMap = _registry.getColorMap()
70+
return {
71+
theme,
72+
colorMap,
73+
}
74+
}
75+
76+
function getLoadedThemes() {
77+
ensureNotDisposed()
78+
return _registry.getLoadedThemes()
79+
}
80+
81+
function getLoadedLanguages() {
82+
ensureNotDisposed()
83+
return _registry.getLoadedLanguages()
84+
}
85+
86+
function loadLanguageSync(...langs: MaybeArray<LanguageRegistration>[]) {
87+
ensureNotDisposed()
88+
_registry.loadLanguages(langs.flat(1))
89+
}
90+
91+
async function loadLanguage(...langs: (LanguageInput | SpecialLanguage)[]) {
92+
return loadLanguageSync(await resolveLangs(langs))
93+
}
94+
95+
async function loadThemeSync(...themes: MaybeArray<ThemeRegistrationAny>[]) {
96+
ensureNotDisposed()
97+
for (const theme of themes.flat(1)) {
98+
_registry.loadTheme(theme)
99+
}
100+
}
101+
102+
async function loadTheme(...themes: (ThemeInput | SpecialTheme)[]) {
103+
ensureNotDisposed()
104+
return loadThemeSync(await resolveThemes(themes))
105+
}
106+
107+
function ensureNotDisposed() {
108+
if (isDisposed)
109+
throw new ShikiError('Shiki instance has been disposed')
110+
}
111+
112+
function dispose() {
113+
if (isDisposed)
114+
return
115+
isDisposed = true
116+
_registry.dispose()
117+
instancesCount -= 1
118+
}
119+
120+
return {
121+
setTheme,
122+
getTheme,
123+
getLanguage,
124+
getLoadedThemes,
125+
getLoadedLanguages,
126+
loadLanguage,
127+
loadLanguageSync,
128+
loadTheme,
129+
loadThemeSync,
130+
dispose,
131+
[Symbol.dispose]: dispose,
132+
}
133+
}

‎packages/core/src/constructors/internal.ts

+11-125
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,11 @@
11
import type {
22
HighlighterCoreOptions,
3-
LanguageInput,
4-
LanguageRegistration,
53
LoadWasmOptions,
6-
MaybeGetter,
74
ShikiInternal,
8-
SpecialLanguage,
9-
SpecialTheme,
10-
ThemeInput,
11-
ThemeRegistrationAny,
12-
ThemeRegistrationResolved,
135
} from '../types'
14-
import { Registry } from '../textmate/registry'
15-
import { Resolver } from '../textmate/resolver'
16-
import { normalizeTheme } from '../textmate/normalize-theme'
17-
import { isSpecialLang, isSpecialTheme } from '../utils'
18-
import { ShikiError } from '../error'
196
import { createWasmOnigEngine } from '../engines/wasm'
7+
import { resolveLangs, resolveThemes } from '../textmate/getters-resolve'
8+
import { createShikiInternalSync } from './internal-sync'
209

2110
let _defaultWasmLoader: LoadWasmOptions | undefined
2211
/**
@@ -27,130 +16,27 @@ export function setDefaultWasmLoader(_loader: LoadWasmOptions) {
2716
_defaultWasmLoader = _loader
2817
}
2918

30-
let instancesCount = 0
31-
3219
/**
3320
* Get the minimal shiki context for rendering.
3421
*/
3522
export async function createShikiInternal(options: HighlighterCoreOptions = {}): Promise<ShikiInternal> {
36-
instancesCount += 1
37-
if (options.warnings !== false && instancesCount >= 10 && instancesCount % 10 === 0)
38-
console.warn(`[Shiki] ${instancesCount} instances have been created. Shiki is supposed to be used as a singleton, consider refactoring your code to cache your highlighter instance; Or call \`highlighter.dispose()\` to release unused instances.`)
39-
40-
let isDisposed = false
41-
42-
async function normalizeGetter<T>(p: MaybeGetter<T>): Promise<T> {
43-
return Promise.resolve(typeof p === 'function' ? (p as any)() : p).then(r => r.default || r)
44-
}
45-
46-
async function resolveLangs(langs: (LanguageInput | SpecialLanguage)[]) {
47-
return Array.from(new Set((await Promise.all(
48-
langs
49-
.filter(l => !isSpecialLang(l))
50-
.map(async lang => await normalizeGetter(lang as LanguageInput).then(r => Array.isArray(r) ? r : [r])),
51-
)).flat()))
52-
}
53-
5423
const [
5524
themes,
5625
langs,
26+
engine,
5727
] = await Promise.all([
58-
Promise.all((options.themes || []).map(normalizeGetter)).then(r => r.map(normalizeTheme)),
28+
resolveThemes(options.themes || []),
5929
resolveLangs(options.langs || []),
30+
(options.engine || createWasmOnigEngine(options.loadWasm || _defaultWasmLoader)),
6031
] as const)
6132

62-
const resolver = new Resolver(
63-
await (options.engine || createWasmOnigEngine(options.loadWasm || _defaultWasmLoader)),
33+
return createShikiInternalSync({
34+
...options,
35+
loadWasm: undefined,
36+
themes,
6437
langs,
65-
)
66-
67-
const _registry = new Registry(resolver, themes, langs, options.langAlias)
68-
await _registry.init()
69-
70-
let _lastTheme: string | ThemeRegistrationAny
71-
72-
function getLanguage(name: string | LanguageRegistration) {
73-
ensureNotDisposed()
74-
const _lang = _registry.getGrammar(typeof name === 'string' ? name : name.name)
75-
if (!_lang)
76-
throw new ShikiError(`Language \`${name}\` not found, you may need to load it first`)
77-
return _lang
78-
}
79-
80-
function getTheme(name: string | ThemeRegistrationAny): ThemeRegistrationResolved {
81-
if (name === 'none')
82-
return { bg: '', fg: '', name: 'none', settings: [], type: 'dark' }
83-
ensureNotDisposed()
84-
const _theme = _registry.getTheme(name)
85-
if (!_theme)
86-
throw new ShikiError(`Theme \`${name}\` not found, you may need to load it first`)
87-
return _theme
88-
}
89-
90-
function setTheme(name: string | ThemeRegistrationAny) {
91-
ensureNotDisposed()
92-
const theme = getTheme(name)
93-
if (_lastTheme !== name) {
94-
_registry.setTheme(theme)
95-
_lastTheme = name
96-
}
97-
const colorMap = _registry.getColorMap()
98-
return {
99-
theme,
100-
colorMap,
101-
}
102-
}
103-
104-
function getLoadedThemes() {
105-
ensureNotDisposed()
106-
return _registry.getLoadedThemes()
107-
}
108-
109-
function getLoadedLanguages() {
110-
ensureNotDisposed()
111-
return _registry.getLoadedLanguages()
112-
}
113-
114-
async function loadLanguage(...langs: (LanguageInput | SpecialLanguage)[]) {
115-
ensureNotDisposed()
116-
await _registry.loadLanguages(await resolveLangs(langs))
117-
}
118-
119-
async function loadTheme(...themes: (ThemeInput | SpecialTheme)[]) {
120-
ensureNotDisposed()
121-
await Promise.all(
122-
themes.map(async theme =>
123-
isSpecialTheme(theme)
124-
? null
125-
: _registry.loadTheme(await normalizeGetter(theme)),
126-
),
127-
)
128-
}
129-
130-
function ensureNotDisposed() {
131-
if (isDisposed)
132-
throw new ShikiError('Shiki instance has been disposed')
133-
}
134-
135-
function dispose() {
136-
if (isDisposed)
137-
return
138-
isDisposed = true
139-
_registry.dispose()
140-
instancesCount -= 1
141-
}
142-
143-
return {
144-
setTheme,
145-
getTheme,
146-
getLanguage,
147-
getLoadedThemes,
148-
getLoadedLanguages,
149-
loadLanguage,
150-
loadTheme,
151-
dispose,
152-
[Symbol.dispose]: dispose,
153-
}
38+
engine,
39+
})
15440
}
15541

15642
/**

‎packages/core/src/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ export * from './types'
44
// Constructors
55
export * from './constructors/highlighter'
66
export * from './constructors/bundle-factory'
7+
export { createShikiInternalSync } from './constructors/internal-sync'
78
export { createShikiInternal, getShikiInternal, setDefaultWasmLoader } from './constructors/internal'
89

910
// Engines

‎packages/core/src/textmate/getters-resolve.ts

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import type { LanguageInput, LanguageRegistration, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationResolved } from '../types'
2+
import { isSpecialLang, isSpecialTheme, normalizeGetter } from '../utils'
3+
import { normalizeTheme } from './normalize-theme'
4+
5+
/**
6+
* Resolve
7+
*/
8+
export async function resolveLangs(langs: (LanguageInput | SpecialLanguage)[]): Promise<LanguageRegistration[]> {
9+
return Array.from(new Set((await Promise.all(
10+
langs
11+
.filter(l => !isSpecialLang(l))
12+
.map(async lang => await normalizeGetter(lang as LanguageInput).then(r => Array.isArray(r) ? r : [r])),
13+
)).flat()))
14+
}
15+
16+
export async function resolveThemes(themes: (ThemeInput | SpecialTheme)[]): Promise<ThemeRegistrationResolved[]> {
17+
const resolved = await Promise.all(
18+
themes.map(async theme =>
19+
isSpecialTheme(theme)
20+
? null
21+
: normalizeTheme(await normalizeGetter(theme)),
22+
),
23+
)
24+
return resolved.filter(i => !!i)
25+
}

‎packages/core/src/textmate/registry.ts

+7-12
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ export class Registry extends TextMateRegistry {
2323
) {
2424
super(_resolver)
2525

26-
_themes.forEach(t => this.loadTheme(t))
27-
_langs.forEach(l => this.loadLanguage(l))
26+
this._themes.map(t => this.loadTheme(t))
27+
this.loadLanguages(this._langs)
2828
}
2929

3030
public getTheme(theme: ThemeRegistrationAny | string) {
@@ -79,7 +79,7 @@ export class Registry extends TextMateRegistry {
7979
return this._resolvedGrammars.get(name)
8080
}
8181

82-
public async loadLanguage(lang: LanguageRegistration) {
82+
public loadLanguage(lang: LanguageRegistration): void {
8383
if (this.getGrammar(lang.name))
8484
return
8585

@@ -97,7 +97,7 @@ export class Registry extends TextMateRegistry {
9797

9898
// @ts-expect-error Private members, set this to override the previous grammar (that can be a stub)
9999
this._syncRegistry._rawGrammars.set(lang.scopeName, lang)
100-
const g = await this.loadGrammarWithConfiguration(lang.scopeName, 1, grammarConfig) as Grammar
100+
const g = this.loadGrammarWithConfiguration(lang.scopeName, 1, grammarConfig) as Grammar
101101
g.name = lang.name
102102
this._resolvedGrammars.set(lang.name, g)
103103
if (lang.aliases) {
@@ -118,16 +118,11 @@ export class Registry extends TextMateRegistry {
118118
this._syncRegistry?._injectionGrammars?.delete(e.scopeName)
119119
// @ts-expect-error clear cache
120120
this._syncRegistry?._grammars?.delete(e.scopeName)
121-
await this.loadLanguage(this._langMap.get(e.name)!)
121+
this.loadLanguage(this._langMap.get(e.name)!)
122122
}
123123
}
124124
}
125125

126-
async init() {
127-
this._themes.map(t => this.loadTheme(t))
128-
await this.loadLanguages(this._langs)
129-
}
130-
131126
public override dispose(): void {
132127
super.dispose()
133128
this._resolvedThemes.clear()
@@ -137,7 +132,7 @@ export class Registry extends TextMateRegistry {
137132
this._loadedThemesCache = null
138133
}
139134

140-
public async loadLanguages(langs: LanguageRegistration[]) {
135+
public loadLanguages(langs: LanguageRegistration[]) {
141136
for (const lang of langs)
142137
this.resolveEmbeddedLanguages(lang)
143138

@@ -155,7 +150,7 @@ export class Registry extends TextMateRegistry {
155150
this._resolver.addLanguage(lang)
156151

157152
for (const [_, lang] of langsGraphArray)
158-
await this.loadLanguage(lang)
153+
this.loadLanguage(lang)
159154
}
160155

161156
public getLoadedLanguages() {

‎packages/core/src/types/engines.ts

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { OnigScanner, OnigString } from '@shikijs/vscode-textmate'
2+
import type { Awaitable } from './utils'
23

34
export interface PatternScanner extends OnigScanner {}
45

@@ -12,8 +13,6 @@ export interface RegexEngine {
1213
createString: (s: string) => RegexEngineString
1314
}
1415

15-
type Awaitable<T> = T | Promise<T>
16-
1716
export interface WebAssemblyInstantiator {
1817
(importObject: Record<string, Record<string, WebAssembly.ImportValue>> | undefined): Promise<WebAssemblyInstance>
1918
}

‎packages/core/src/types/highlighter.ts

+10
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { LanguageInput, LanguageRegistration, ResolveBundleKey, SpecialLang
44
import type { SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved } from './themes'
55
import type { CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, GrammarState, ThemedToken, ThemedTokenWithVariants, TokensResult } from './tokens'
66
import type { CodeToHastOptions } from './options'
7+
import type { MaybeArray } from './utils'
78

89
/**
910
* Internal context of Shiki, core textmate logic
@@ -13,10 +14,19 @@ export interface ShikiInternal<BundledLangKeys extends string = never, BundledTh
1314
* Load a theme to the highlighter, so later it can be used synchronously.
1415
*/
1516
loadTheme: (...themes: (ThemeInput | BundledThemeKeys | SpecialTheme)[]) => Promise<void>
17+
/**
18+
* Load a theme registration synchronously.
19+
*/
20+
loadThemeSync: (...themes: MaybeArray<ThemeRegistrationAny>[]) => void
21+
1622
/**
1723
* Load a language to the highlighter, so later it can be used synchronously.
1824
*/
1925
loadLanguage: (...langs: (LanguageInput | BundledLangKeys | SpecialLanguage)[]) => Promise<void>
26+
/**
27+
* Load a language registration synchronously.
28+
*/
29+
loadLanguageSync: (...langs: MaybeArray<LanguageRegistration>[]) => void
2030

2131
/**
2232
* Get the registered theme object

‎packages/core/src/types/options.ts

+13-11
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,40 @@
11
import type { LoadWasmOptions, RegexEngine } from '../types'
2-
import type { StringLiteralUnion } from './utils'
3-
import type { LanguageInput, SpecialLanguage } from './langs'
2+
import type { Awaitable, MaybeArray, StringLiteralUnion } from './utils'
3+
import type { LanguageInput, LanguageRegistration, SpecialLanguage } from './langs'
44
import type { SpecialTheme, ThemeInput, ThemeRegistrationAny } from './themes'
55
import type { TransformerOptions } from './transformers'
66
import type { TokenizeWithThemeOptions, TokensResult } from './tokens'
77
import type { DecorationOptions } from './decorations'
88

9-
export interface HighlighterCoreOptions {
9+
export interface HighlighterCoreOptions<Sync extends boolean = false> {
10+
/**
11+
* Custom RegExp engine.
12+
*/
13+
engine?: Sync extends true ? RegexEngine : Awaitable<RegexEngine>
1014
/**
1115
* Theme names, or theme registration objects to be loaded upfront.
1216
*/
13-
themes?: ThemeInput[]
17+
themes?: Sync extends true ? MaybeArray<ThemeRegistrationAny>[] : ThemeInput[]
1418
/**
1519
* Language names, or language registration objects to be loaded upfront.
1620
*/
17-
langs?: LanguageInput[]
21+
langs?: Sync extends true ? MaybeArray<LanguageRegistration>[] : LanguageInput[]
1822
/**
1923
* Alias of languages
2024
* @example { 'my-lang': 'javascript' }
2125
*/
2226
langAlias?: Record<string, string>
23-
/**
24-
* Load wasm file from a custom path or using a custom function.
25-
*/
26-
loadWasm?: LoadWasmOptions
2727
/**
2828
* Emit console warnings to alert users of potential issues.
2929
* @default true
3030
*/
3131
warnings?: boolean
32+
33+
// TODO: Deprecate this option after docs for engines are updated.
3234
/**
33-
* Custom RegExp engine.
35+
* Load wasm file from a custom path or using a custom function.
3436
*/
35-
engine?: RegexEngine | Promise<RegexEngine>
37+
loadWasm?: Sync extends true ? never : LoadWasmOptions
3638
}
3739

3840
export interface BundledHighlighterOptions<L extends string, T extends string> extends Pick<HighlighterCoreOptions, 'warnings' | 'engine'> {

‎packages/core/src/utils.ts

+20-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,18 @@
11
import type { Element } from 'hast'
22
import { FontStyle } from './types'
3-
import type { MaybeArray, PlainTextLanguage, Position, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemedToken, TokenStyles, TokenizeWithThemeOptions } from './types'
3+
import type {
4+
MaybeArray,
5+
MaybeGetter,
6+
PlainTextLanguage,
7+
Position,
8+
SpecialLanguage,
9+
SpecialTheme,
10+
ThemeInput,
11+
ThemeRegistrationAny,
12+
ThemedToken,
13+
TokenStyles,
14+
TokenizeWithThemeOptions,
15+
} from './types'
416

517
export function toArray<T>(x: MaybeArray<T>): T[] {
618
return Array.isArray(x) ? x : [x]
@@ -146,6 +158,13 @@ export function splitTokens<
146158
})
147159
}
148160

161+
/**
162+
* Normalize a getter to a promise.
163+
*/
164+
export async function normalizeGetter<T>(p: MaybeGetter<T>): Promise<T> {
165+
return Promise.resolve(typeof p === 'function' ? (p as any)() : p).then(r => r.default || r)
166+
}
167+
149168
export function resolveColorReplacements(
150169
theme: ThemeRegistrationAny | string,
151170
options?: TokenizeWithThemeOptions,

‎packages/markdown-it/test/index.test.ts

+5-3
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ it('run for base', async () => {
1919
const result = md.render(await fs.readFile(new URL('./fixtures/a.md', import.meta.url), 'utf-8'))
2020

2121
expect(result).toMatchFileSnapshot('./fixtures/a.out.html')
22-
})
22+
}, { timeout: 10_000 })
23+
2324
it('run for fallback language', async () => {
2425
const md = MarkdownIt()
2526
md.use(await Shiki({
@@ -36,7 +37,8 @@ it('run for fallback language', async () => {
3637
const result = md.render(await fs.readFile(new URL('./fixtures/b.md', import.meta.url), 'utf-8'))
3738

3839
expect(result).toMatchFileSnapshot('./fixtures/b.out.html')
39-
})
40+
}, { timeout: 10_000 })
41+
4042
it('run for default language', async () => {
4143
const md = MarkdownIt()
4244
md.use(await Shiki({
@@ -53,4 +55,4 @@ it('run for default language', async () => {
5355
const result = md.render(await fs.readFile(new URL('./fixtures/c.md', import.meta.url), 'utf-8'))
5456

5557
expect(result).toMatchFileSnapshot('./fixtures/c.out.html')
56-
})
58+
}, { timeout: 10_000 })

‎packages/shiki/test/core-sync.test.ts

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import { describe, expect, it } from 'vitest'
2+
import { createHighlighterCoreSync, createJavaScriptRegexEngine } from '../src/core'
3+
4+
import js from '../src/assets/langs/javascript'
5+
import nord from '../src/assets/themes/nord'
6+
7+
describe('should', () => {
8+
const engine = createJavaScriptRegexEngine()
9+
10+
it('works', () => {
11+
const shiki = createHighlighterCoreSync({
12+
themes: [nord],
13+
langs: [js],
14+
engine,
15+
})
16+
17+
expect(shiki.codeToHtml('console.log("Hi")', { lang: 'javascript', theme: 'nord' }))
18+
.toMatchInlineSnapshot(`"<pre class="shiki nord" style="background-color:#2e3440ff;color:#d8dee9ff" tabindex="0"><code><span class="line"><span style="color:#D8DEE9">console</span><span style="color:#ECEFF4">.</span><span style="color:#88C0D0">log</span><span style="color:#D8DEE9FF">(</span><span style="color:#ECEFF4">"</span><span style="color:#A3BE8C">Hi</span><span style="color:#ECEFF4">"</span><span style="color:#D8DEE9FF">)</span></span></code></pre>"`)
19+
})
20+
21+
it('dynamic load sync theme and lang', async () => {
22+
const shiki = createHighlighterCoreSync({
23+
themes: [nord],
24+
langs: [
25+
js,
26+
// Load the grammar upfront (await outside of the function)
27+
await import('../src/assets/langs/c').then(r => r.default),
28+
],
29+
engine,
30+
})
31+
32+
shiki.loadLanguageSync(await import('../src/assets/langs/python').then(m => m.default))
33+
shiki.loadThemeSync(await import('../dist/themes/vitesse-light.mjs').then(m => m.default))
34+
35+
expect(shiki.getLoadedLanguages())
36+
.toMatchInlineSnapshot(`
37+
[
38+
"javascript",
39+
"c",
40+
"python",
41+
"js",
42+
"py",
43+
]
44+
`)
45+
expect(shiki.getLoadedThemes())
46+
.toMatchInlineSnapshot(`
47+
[
48+
"nord",
49+
"vitesse-light",
50+
]
51+
`)
52+
53+
expect(shiki.codeToHtml('print 1', { lang: 'python', theme: 'vitesse-light' }))
54+
.toMatchInlineSnapshot(`"<pre class="shiki vitesse-light" style="background-color:#ffffff;color:#393a34" tabindex="0"><code><span class="line"><span style="color:#998418">print</span><span style="color:#2F798A"> 1</span></span></code></pre>"`)
55+
})
56+
})

‎packages/shiki/test/core.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ describe('should', () => {
1313
it('works', async () => {
1414
const shiki = await createHighlighterCore({
1515
themes: [nord],
16-
langs: [js as any],
16+
langs: [js],
1717
loadWasm: {
1818
instantiator: obj => WebAssembly.instantiate(wasmBinary, obj),
1919
},

0 commit comments

Comments
 (0)
Please sign in to comment.