From 56d1a486e39bd9cc8432cfa2c2a51b3ccfae3eda Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Fri, 15 Mar 2024 16:26:44 +0100 Subject: [PATCH] perf: bundle wasm binary in chunk for `shiki/wasm` and `@shikijs/core/wasm-inlined` (#624) --- docs/packages/markdown-it.md | 3 +-- docs/packages/rehype.md | 3 +-- packages/core/rollup.config.mjs | 4 +--- packages/core/src/types/index.ts | 2 ++ packages/core/src/wasm-inlined.ts | 12 +++++++----- packages/shiki/src/bundle-full.ts | 7 ++++--- packages/shiki/src/bundle-web.ts | 7 ++++--- packages/shiki/src/wasm-dynamic.ts | 5 +++++ packages/shiki/test/core.test.ts | 8 ++++---- packages/shiki/test/wasm1.test.ts | 4 ++-- packages/shiki/test/wasm2.test.ts | 4 ++-- packages/shiki/test/wasm3.test.ts | 4 ++-- packages/shiki/test/wasm4.test.ts | 4 ++-- tsconfig.json | 1 + 14 files changed, 38 insertions(+), 30 deletions(-) create mode 100644 packages/shiki/src/wasm-dynamic.ts diff --git a/docs/packages/markdown-it.md b/docs/packages/markdown-it.md index b4c682c71..f7e10c88a 100644 --- a/docs/packages/markdown-it.md +++ b/docs/packages/markdown-it.md @@ -35,7 +35,6 @@ By default, the full bundle of `shiki` will be imported. If you are using a [fin import MarkdownIt from 'markdown-it' import { fromHighlighter } from '@shikijs/markdown-it/core' import { getHighlighterCore } from 'shiki/core' -import getWasm from 'shiki/wasm' const highlighter = await getHighlighterCore({ themes: [ @@ -44,7 +43,7 @@ const highlighter = await getHighlighterCore({ langs: [ import('shiki/langs/javascript.mjs'), ], - loadWasm: getWasm + loadWasm: import('shiki/wasm') }) const md = MarkdownIt() diff --git a/docs/packages/rehype.md b/docs/packages/rehype.md index 0ffedd8e1..5cfe1b161 100644 --- a/docs/packages/rehype.md +++ b/docs/packages/rehype.md @@ -51,7 +51,6 @@ import rehypeStringify from 'rehype-stringify' import rehypeShikiFromHighlighter from '@shikijs/rehype/core' import { getHighlighterCore } from 'shiki/core' -import getWasm from 'shiki/wasm' const highlighter = await getHighlighterCore({ themes: [ @@ -60,7 +59,7 @@ const highlighter = await getHighlighterCore({ langs: [ import('shiki/langs/javascript.mjs'), ], - loadWasm: getWasm + loadWasm: import('shiki/wasm') }) const raw = await fs.readFile('./input.md') diff --git a/packages/core/rollup.config.mjs b/packages/core/rollup.config.mjs index ee8cd9ad8..1d25343ba 100644 --- a/packages/core/rollup.config.mjs +++ b/packages/core/rollup.config.mjs @@ -40,9 +40,7 @@ export default defineConfig([ dir: 'dist', format: 'esm', entryFileNames: '[name].mjs', - chunkFileNames: (f) => { - if (f.name === 'onig') - return 'onig.mjs' + chunkFileNames: () => { return 'chunks-[name].mjs' }, }, diff --git a/packages/core/src/types/index.ts b/packages/core/src/types/index.ts index 617d5f532..f089d8d20 100644 --- a/packages/core/src/types/index.ts +++ b/packages/core/src/types/index.ts @@ -7,3 +7,5 @@ export * from './tokens' export * from './transformers' export * from './utils' export * from './decorations' + +export { WebAssemblyInstantiator } from '../oniguruma' diff --git a/packages/core/src/wasm-inlined.ts b/packages/core/src/wasm-inlined.ts index f4f2cc3ac..da82fe565 100644 --- a/packages/core/src/wasm-inlined.ts +++ b/packages/core/src/wasm-inlined.ts @@ -1,9 +1,11 @@ +// @ts-expect-error this will be compiled to ArrayBuffer +import binary from 'vscode-oniguruma/release/onig.wasm' import type { WebAssemblyInstantiator } from './oniguruma' -const getWasm: WebAssemblyInstantiator = async (info) => { - // @ts-expect-error this will be compiled to ArrayBuffer - const binray: ArrayBuffer = await import('vscode-oniguruma/release/onig.wasm').then(m => m.default) - return WebAssembly.instantiate(binray, info).then(wasm => wasm.instance.exports) +export const wasmBinary = binary as ArrayBuffer + +export const getWasmInstance: WebAssemblyInstantiator = async (info) => { + return WebAssembly.instantiate(wasmBinary, info).then(wasm => wasm.instance.exports) } -export default getWasm +export default getWasmInstance diff --git a/packages/shiki/src/bundle-full.ts b/packages/shiki/src/bundle-full.ts index c3fe5f363..df368d35f 100644 --- a/packages/shiki/src/bundle-full.ts +++ b/packages/shiki/src/bundle-full.ts @@ -1,15 +1,16 @@ import type { HighlighterGeneric } from '@shikijs/core' -import getWasm from 'shiki/wasm' import { createSingletonShorthands, createdBundledHighlighter } from './core' import type { BundledLanguage } from './assets/langs-bundle-full' import type { BundledTheme } from './themes' import { bundledLanguages } from './assets/langs-bundle-full' import { bundledThemes } from './themes' +import { getWasmInlined } from './wasm-dynamic' export * from './core' export * from './themes' export * from './assets/langs-bundle-full' -export { default as getWasmInlined } from 'shiki/wasm' + +export { getWasmInlined } export type Highlighter = HighlighterGeneric @@ -29,7 +30,7 @@ export const getHighlighter = /* @__PURE__ */ createdBundledHighlighter< >( bundledLanguages, bundledThemes, - getWasm, + getWasmInlined, ) export const { diff --git a/packages/shiki/src/bundle-web.ts b/packages/shiki/src/bundle-web.ts index 7ba2c2322..7c399e06e 100644 --- a/packages/shiki/src/bundle-web.ts +++ b/packages/shiki/src/bundle-web.ts @@ -1,15 +1,16 @@ import type { HighlighterGeneric } from '@shikijs/core' -import getWasm from 'shiki/wasm' import { createSingletonShorthands, createdBundledHighlighter } from './core' import type { BundledLanguage } from './assets/langs-bundle-web' import type { BundledTheme } from './themes' import { bundledLanguages } from './assets/langs-bundle-web' import { bundledThemes } from './themes' +import { getWasmInlined } from './wasm-dynamic' export * from './core' export * from './themes' export * from './assets/langs-bundle-web' -export { default as getWasmInlined } from 'shiki/wasm' + +export { getWasmInlined } export type Highlighter = HighlighterGeneric @@ -29,7 +30,7 @@ export const getHighlighter = /* @__PURE__ */ createdBundledHighlighter< >( bundledLanguages, bundledThemes, - getWasm, + getWasmInlined, ) export const { diff --git a/packages/shiki/src/wasm-dynamic.ts b/packages/shiki/src/wasm-dynamic.ts new file mode 100644 index 000000000..d7557bd5b --- /dev/null +++ b/packages/shiki/src/wasm-dynamic.ts @@ -0,0 +1,5 @@ +import type { WebAssemblyInstantiator } from './types' + +export const getWasmInlined: WebAssemblyInstantiator = async (info) => { + return import('shiki/wasm').then(wasm => wasm.default(info)) +} diff --git a/packages/shiki/test/core.test.ts b/packages/shiki/test/core.test.ts index 641772533..a5a8dbce8 100644 --- a/packages/shiki/test/core.test.ts +++ b/packages/shiki/test/core.test.ts @@ -7,7 +7,7 @@ import nord from '../src/assets/themes/nord' import mtp from '../src/assets/themes/material-theme-palenight' // eslint-disable-next-line antfu/no-import-dist -import onig from '../../core/dist/onig.mjs' +import { wasmBinary } from '../../core/dist/wasm-inlined.mjs' describe('should', () => { it('works', async () => { @@ -15,7 +15,7 @@ describe('should', () => { themes: [nord], langs: [js as any], loadWasm: { - instantiator: obj => WebAssembly.instantiate(onig, obj), + instantiator: obj => WebAssembly.instantiate(wasmBinary, obj), }, }) @@ -32,7 +32,7 @@ describe('should', () => { ], loadWasm: { // https://github.com/WebAssembly/esm-integration/tree/main/proposals/esm-integration - instantiator: obj => WebAssembly.instantiate(onig, obj).then(r => r.instance.exports), + instantiator: obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance.exports), }, }) @@ -151,7 +151,7 @@ describe('errors', () => { themes: [nord], langs: [js as any], loadWasm: { - instantiator: obj => WebAssembly.instantiate(onig, obj), + instantiator: obj => WebAssembly.instantiate(wasmBinary, obj), }, }) diff --git a/packages/shiki/test/wasm1.test.ts b/packages/shiki/test/wasm1.test.ts index d46a2beed..1a108eb21 100644 --- a/packages/shiki/test/wasm1.test.ts +++ b/packages/shiki/test/wasm1.test.ts @@ -5,14 +5,14 @@ import js from '../src/assets/langs/javascript' import nord from '../src/assets/themes/nord' // eslint-disable-next-line antfu/no-import-dist -import onig from '../../core/dist/onig.mjs' +import { wasmBinary } from '../../core/dist/wasm-inlined.mjs' it('wasm', async () => { const shiki = await getHighlighterCore({ themes: [nord], langs: [js as any], loadWasm: { - instantiator: obj => WebAssembly.instantiate(onig, obj), + instantiator: obj => WebAssembly.instantiate(wasmBinary, obj), }, }) diff --git a/packages/shiki/test/wasm2.test.ts b/packages/shiki/test/wasm2.test.ts index b66ccbd6b..6ef48d9af 100644 --- a/packages/shiki/test/wasm2.test.ts +++ b/packages/shiki/test/wasm2.test.ts @@ -5,14 +5,14 @@ import js from '../src/assets/langs/javascript' import nord from '../src/assets/themes/nord' // eslint-disable-next-line antfu/no-import-dist -import onig from '../../core/dist/onig.mjs' +import { wasmBinary } from '../../core/dist/wasm-inlined.mjs' it('wasm', async () => { const shiki = await getHighlighterCore({ themes: [nord], langs: [js as any], loadWasm: { - default: obj => WebAssembly.instantiate(onig, obj).then(r => r.instance.exports), + default: obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance.exports), }, }) diff --git a/packages/shiki/test/wasm3.test.ts b/packages/shiki/test/wasm3.test.ts index 66006ef19..78973ae65 100644 --- a/packages/shiki/test/wasm3.test.ts +++ b/packages/shiki/test/wasm3.test.ts @@ -5,13 +5,13 @@ import js from '../src/assets/langs/javascript' import nord from '../src/assets/themes/nord' // eslint-disable-next-line antfu/no-import-dist -import onig from '../../core/dist/onig.mjs' +import { wasmBinary } from '../../core/dist/wasm-inlined.mjs' it('wasm', async () => { const shiki = await getHighlighterCore({ themes: [nord], langs: [js as any], - loadWasm: obj => WebAssembly.instantiate(onig, obj).then(r => r.instance), + loadWasm: obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance), }) expect(shiki.codeToHtml('1 + 1', { lang: 'javascript', theme: 'nord' })) diff --git a/packages/shiki/test/wasm4.test.ts b/packages/shiki/test/wasm4.test.ts index a0c4cb238..49bd3d532 100644 --- a/packages/shiki/test/wasm4.test.ts +++ b/packages/shiki/test/wasm4.test.ts @@ -5,13 +5,13 @@ import js from '../src/assets/langs/javascript' import nord from '../src/assets/themes/nord' // eslint-disable-next-line antfu/no-import-dist -import onig from '../../core/dist/onig.mjs' +import { wasmBinary } from '../../core/dist/wasm-inlined.mjs' it('wasm', async () => { const shiki = await getHighlighterCore({ themes: [nord], langs: [js as any], - loadWasm: Promise.resolve().then(() => obj => WebAssembly.instantiate(onig, obj).then(r => r.instance)), + loadWasm: Promise.resolve().then(() => obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance)), }) expect(shiki.codeToHtml('1 + 1', { lang: 'javascript', theme: 'nord' })) diff --git a/tsconfig.json b/tsconfig.json index 43742d743..f5745a346 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,6 +29,7 @@ "allowJs": true, "strict": true, "strictNullChecks": true, + "noEmit": true, "preserveValueImports": false, "esModuleInterop": true, "skipDefaultLibCheck": true,