perf: bundle wasm binary in chunk for shiki/wasm and @shikijs/core/wasm-inlined (#624)

Author: antfu

docs/packages/markdown-it.md

@@ -35,7 +35,6 @@ By default, the full bundle of `shiki` will be imported. If you are using a [fin
 import MarkdownIt from 'markdown-it'
 import { fromHighlighter } from '@shikijs/markdown-it/core'
 import { getHighlighterCore } from 'shiki/core'
-import getWasm from 'shiki/wasm'
 
 const highlighter = await getHighlighterCore({
   themes: [
@@ -44,7 +43,7 @@ const highlighter = await getHighlighterCore({
   langs: [
     import('shiki/langs/javascript.mjs'),
   ],
-  loadWasm: getWasm
+  loadWasm: import('shiki/wasm')
 })
 
 const md = MarkdownIt()

docs/packages/rehype.md

@@ -51,7 +51,6 @@ import rehypeStringify from 'rehype-stringify'
 import rehypeShikiFromHighlighter from '@shikijs/rehype/core'
 
 import { getHighlighterCore } from 'shiki/core'
-import getWasm from 'shiki/wasm'
 
 const highlighter = await getHighlighterCore({
   themes: [
@@ -60,7 +59,7 @@ const highlighter = await getHighlighterCore({
   langs: [
     import('shiki/langs/javascript.mjs'),
   ],
-  loadWasm: getWasm
+  loadWasm: import('shiki/wasm')
 })
 
 const raw = await fs.readFile('./input.md')

packages/core/rollup.config.mjs

@@ -40,9 +40,7 @@ export default defineConfig([
       dir: 'dist',
       format: 'esm',
       entryFileNames: '[name].mjs',
-      chunkFileNames: (f) => {
-        if (f.name === 'onig')
-          return 'onig.mjs'
+      chunkFileNames: () => {
         return 'chunks-[name].mjs'
       },
     },

packages/core/src/types/index.ts

@@ -7,3 +7,5 @@ export * from './tokens'
 export * from './transformers'
 export * from './utils'
 export * from './decorations'
+
+export { WebAssemblyInstantiator } from '../oniguruma'

packages/core/src/wasm-inlined.ts

@@ -1,9 +1,11 @@
+// @ts-expect-error this will be compiled to ArrayBuffer
+import binary from 'vscode-oniguruma/release/onig.wasm'
 import type { WebAssemblyInstantiator } from './oniguruma'
 
-const getWasm: WebAssemblyInstantiator = async (info) => {
-  // @ts-expect-error this will be compiled to ArrayBuffer
-  const binray: ArrayBuffer = await import('vscode-oniguruma/release/onig.wasm').then(m => m.default)
-  return WebAssembly.instantiate(binray, info).then(wasm => wasm.instance.exports)
+export const wasmBinary = binary as ArrayBuffer
+
+export const getWasmInstance: WebAssemblyInstantiator = async (info) => {
+  return WebAssembly.instantiate(wasmBinary, info).then(wasm => wasm.instance.exports)
 }
 
-export default getWasm
+export default getWasmInstance

packages/shiki/src/bundle-full.ts

@@ -1,15 +1,16 @@
 import type { HighlighterGeneric } from '@shikijs/core'
-import getWasm from 'shiki/wasm'
 import { createSingletonShorthands, createdBundledHighlighter } from './core'
 import type { BundledLanguage } from './assets/langs-bundle-full'
 import type { BundledTheme } from './themes'
 import { bundledLanguages } from './assets/langs-bundle-full'
 import { bundledThemes } from './themes'
+import { getWasmInlined } from './wasm-dynamic'
 
 export * from './core'
 export * from './themes'
 export * from './assets/langs-bundle-full'
-export { default as getWasmInlined } from 'shiki/wasm'
+
+export { getWasmInlined }
 
 export type Highlighter = HighlighterGeneric<BundledLanguage, BundledTheme>
 
@@ -29,7 +30,7 @@ export const getHighlighter = /* @__PURE__ */ createdBundledHighlighter<
 >(
   bundledLanguages,
   bundledThemes,
-  getWasm,
+  getWasmInlined,
 )
 
 export const {

packages/shiki/src/bundle-web.ts

@@ -1,15 +1,16 @@
 import type { HighlighterGeneric } from '@shikijs/core'
-import getWasm from 'shiki/wasm'
 import { createSingletonShorthands, createdBundledHighlighter } from './core'
 import type { BundledLanguage } from './assets/langs-bundle-web'
 import type { BundledTheme } from './themes'
 import { bundledLanguages } from './assets/langs-bundle-web'
 import { bundledThemes } from './themes'
+import { getWasmInlined } from './wasm-dynamic'
 
 export * from './core'
 export * from './themes'
 export * from './assets/langs-bundle-web'
-export { default as getWasmInlined } from 'shiki/wasm'
+
+export { getWasmInlined }
 
 export type Highlighter = HighlighterGeneric<BundledLanguage, BundledTheme>
 
@@ -29,7 +30,7 @@ export const getHighlighter = /* @__PURE__ */ createdBundledHighlighter<
 >(
   bundledLanguages,
   bundledThemes,
-  getWasm,
+  getWasmInlined,
 )
 
 export const {

packages/shiki/src/wasm-dynamic.ts

@@ -0,0 +1,5 @@
+import type { WebAssemblyInstantiator } from './types'
+
+export const getWasmInlined: WebAssemblyInstantiator = async (info) => {
+  return import('shiki/wasm').then(wasm => wasm.default(info))
+}

packages/shiki/test/core.test.ts

@@ -7,15 +7,15 @@ import nord from '../src/assets/themes/nord'
 import mtp from '../src/assets/themes/material-theme-palenight'
 
 // eslint-disable-next-line antfu/no-import-dist
-import onig from '../../core/dist/onig.mjs'
+import { wasmBinary } from '../../core/dist/wasm-inlined.mjs'
 
 describe('should', () => {
   it('works', async () => {
     const shiki = await getHighlighterCore({
       themes: [nord],
       langs: [js as any],
       loadWasm: {
-        instantiator: obj => WebAssembly.instantiate(onig, obj),
+        instantiator: obj => WebAssembly.instantiate(wasmBinary, obj),
       },
     })
 
@@ -32,7 +32,7 @@ describe('should', () => {
       ],
       loadWasm: {
         // https://github.com/WebAssembly/esm-integration/tree/main/proposals/esm-integration
-        instantiator: obj => WebAssembly.instantiate(onig, obj).then(r => r.instance.exports),
+        instantiator: obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance.exports),
       },
     })
 
@@ -151,7 +151,7 @@ describe('errors', () => {
       themes: [nord],
       langs: [js as any],
       loadWasm: {
-        instantiator: obj => WebAssembly.instantiate(onig, obj),
+        instantiator: obj => WebAssembly.instantiate(wasmBinary, obj),
       },
     })
 

packages/shiki/test/wasm1.test.ts

@@ -5,14 +5,14 @@ import js from '../src/assets/langs/javascript'
 import nord from '../src/assets/themes/nord'
 
 // eslint-disable-next-line antfu/no-import-dist
-import onig from '../../core/dist/onig.mjs'
+import { wasmBinary } from '../../core/dist/wasm-inlined.mjs'
 
 it('wasm', async () => {
   const shiki = await getHighlighterCore({
     themes: [nord],
     langs: [js as any],
     loadWasm: {
-      instantiator: obj => WebAssembly.instantiate(onig, obj),
+      instantiator: obj => WebAssembly.instantiate(wasmBinary, obj),
     },
   })
 

packages/shiki/test/wasm2.test.ts

@@ -5,14 +5,14 @@ import js from '../src/assets/langs/javascript'
 import nord from '../src/assets/themes/nord'
 
 // eslint-disable-next-line antfu/no-import-dist
-import onig from '../../core/dist/onig.mjs'
+import { wasmBinary } from '../../core/dist/wasm-inlined.mjs'
 
 it('wasm', async () => {
   const shiki = await getHighlighterCore({
     themes: [nord],
     langs: [js as any],
     loadWasm: {
-      default: obj => WebAssembly.instantiate(onig, obj).then(r => r.instance.exports),
+      default: obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance.exports),
     },
   })
 

packages/shiki/test/wasm3.test.ts

@@ -5,13 +5,13 @@ import js from '../src/assets/langs/javascript'
 import nord from '../src/assets/themes/nord'
 
 // eslint-disable-next-line antfu/no-import-dist
-import onig from '../../core/dist/onig.mjs'
+import { wasmBinary } from '../../core/dist/wasm-inlined.mjs'
 
 it('wasm', async () => {
   const shiki = await getHighlighterCore({
     themes: [nord],
     langs: [js as any],
-    loadWasm: obj => WebAssembly.instantiate(onig, obj).then(r => r.instance),
+    loadWasm: obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance),
   })
 
   expect(shiki.codeToHtml('1 + 1', { lang: 'javascript', theme: 'nord' }))

packages/shiki/test/wasm4.test.ts

@@ -5,13 +5,13 @@ import js from '../src/assets/langs/javascript'
 import nord from '../src/assets/themes/nord'
 
 // eslint-disable-next-line antfu/no-import-dist
-import onig from '../../core/dist/onig.mjs'
+import { wasmBinary } from '../../core/dist/wasm-inlined.mjs'
 
 it('wasm', async () => {
   const shiki = await getHighlighterCore({
     themes: [nord],
     langs: [js as any],
-    loadWasm: Promise.resolve().then(() => obj => WebAssembly.instantiate(onig, obj).then(r => r.instance)),
+    loadWasm: Promise.resolve().then(() => obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance)),
   })
 
   expect(shiki.codeToHtml('1 + 1', { lang: 'javascript', theme: 'nord' }))

tsconfig.json

@@ -29,6 +29,7 @@
     "allowJs": true,
     "strict": true,
     "strictNullChecks": true,
+    "noEmit": true,
     "preserveValueImports": false,
     "esModuleInterop": true,
     "skipDefaultLibCheck": true,