Update Shiki

CHANGELOG.md

@@ -6,12 +6,18 @@
 -   Moved from `marked` to `markdown-it` for parsing as marked has moved to an async model which supporting would significantly complicate TypeDoc's rendering code.
     This means that any projects setting `markedOptions` needs to be updated to use `markdownItOptions`.
     Unlike `marked@4`, `markdown-it` pushes lots of functionality to plugins. To use plugins, a JavaScript config file must be used with the `markdownItLoader` option.
+-   Updated Shiki from 0.14 to 1.3. This should mostly be a transparent update which adds another 23 supported languages and 13 supported themes.
 -   Removed deprecated `navigation.fullTree` option.
 -   API: `MapOptionDeclaration.mapError` has been removed.
 -   API: Deprecated `BindOption` decorator has been removed.
 -   API: `DeclarationReflection.indexSignature` has been renamed to `DeclarationReflection.indexSignatures`.
     Note: This also affects JSON serialization. TypeDoc will support JSON output from 0.25 until 0.28.
 
+### Features
+
+-   TypeDoc now has the architecture in place to support localization. No languages besides English
+    are currently shipped in the package, but it is now possible to add support for additional languages, #2475.
+
 ### Bug Fixes
 
 -   TypeDoc now supports objects with multiple index signatures, #2470.

package.json

@@ -27,7 +27,7 @@
     "lunr": "^2.3.9",
     "markdown-it": "^14.1.0",
     "minimatch": "^9.0.4",
-    "shiki": "^0.14.7"
+    "shiki": "^1.3.0"
   },
   "peerDependencies": {
     "typescript": "4.6.x || 4.7.x || 4.8.x || 4.9.x || 5.0.x || 5.1.x || 5.2.x || 5.3.x || 5.4.x"

src/index.ts

@@ -35,6 +35,7 @@ export {
     Renderer,
     DefaultTheme,
     DefaultThemeRenderContext,
+    Slugger,
     UrlMapping,
     Theme,
     PageEvent,
@@ -105,5 +106,12 @@ export {
     SerializeEvent,
 } from "./lib/serialization";
 
+export {
+    type TranslationProxy,
+    type TranslatedString,
+    type TranslatableStrings,
+    Internationalization,
+} from "./lib/internationalization/internationalization";
+
 import TypeScript from "typescript";
 export { TypeScript };

src/lib/application.ts

@@ -46,6 +46,7 @@ import {
     Internationalization,
     type TranslatedString,
 } from "./internationalization/internationalization";
+import { loadShikiMetadata } from "./utils/highlighter";
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const packageInfo = require("../../package.json") as {
@@ -188,6 +189,7 @@ export class Application extends ChildableComponent<
         options: Partial<TypeDocOptions> = {},
         readers: readonly OptionsReader[] = DEFAULT_READERS,
     ): Promise<Application> {
+        await loadShikiMetadata();
         const app = new Application(DETECTOR);
         readers.forEach((r) => app.options.addReader(r));
         app.options.reset();
@@ -217,6 +219,7 @@ export class Application extends ChildableComponent<
         options: Partial<TypeDocOptions> = {},
         readers: readonly OptionsReader[] = DEFAULT_READERS,
     ): Promise<Application> {
+        await loadShikiMetadata();
         const app = new Application(DETECTOR);
         readers.forEach((r) => app.options.addReader(r));
         await app._bootstrap(options);

src/lib/output/index.ts

@@ -6,6 +6,7 @@ export type { RendererHooks } from "./renderer";
 export { Theme } from "./theme";
 export {
     DefaultTheme,
+    Slugger,
     type NavigationElement,
 } from "./themes/default/DefaultTheme";
 export { DefaultThemeRenderContext } from "./themes/default/DefaultThemeRenderContext";

src/lib/output/renderer.ts

@@ -20,7 +20,7 @@ import { RendererComponent } from "./components";
 import { Component, ChildableComponent } from "../utils/component";
 import { Option, EventHooks } from "../utils";
 import { loadHighlighter } from "../utils/highlighter";
-import type { Theme as ShikiTheme } from "shiki";
+import type { BundledTheme as ShikiTheme } from "shiki" with { "resolution-mode": "import" };
 import { Reflection } from "../models";
 import type { JsxElement } from "../utils/jsx.elements";
 import type { DefaultThemeRenderContext } from "./themes/default/DefaultThemeRenderContext";

src/lib/output/themes/MarkedPlugin.tsx

@@ -6,7 +6,7 @@ import { Component, ContextAwareRendererComponent } from "../components";
 import { type RendererEvent, MarkdownEvent, type PageEvent } from "../events";
 import { Option, readFile, copySync, isFile, type Logger } from "../../utils";
 import { highlight, isSupportedLanguage } from "../../utils/highlighter";
-import type { Theme } from "shiki";
+import type { BundledTheme } from "shiki" with { "resolution-mode": "import" };
 import { escapeHtml, getTextContent } from "../../utils/html";
 import type { DefaultTheme } from "..";
 import { Slugger } from "./default/DefaultTheme";
@@ -33,10 +33,10 @@ export class MarkedPlugin extends ContextAwareRendererComponent {
     accessor mediaSource!: string;
 
     @Option("lightHighlightTheme")
-    accessor lightTheme!: Theme;
+    accessor lightTheme!: BundledTheme;
 
     @Option("darkHighlightTheme")
-    accessor darkTheme!: Theme;
+    accessor darkTheme!: BundledTheme;
 
     private parser?: MarkdownIt;
 

src/lib/output/themes/default/DefaultTheme.tsx

@@ -49,6 +49,9 @@ export interface NavigationElement {
     children?: NavigationElement[];
 }
 
+/**
+ * Responsible for getting a unique anchor for elements within a page.
+ */
 export class Slugger {
     private seen = new Map<string, number>();
 

src/lib/utils/highlighter.tsx

@@ -1,74 +1,61 @@
-import { ok as assert } from "assert";
-import { BUNDLED_LANGUAGES, getHighlighter, type Highlighter, type Theme } from "shiki";
-import { unique, zip } from "./array";
+import { ok as assert, ok } from "assert";
+import type { BundledLanguage, BundledTheme, Highlighter, TokenStyles } from "shiki" with { "resolution-mode": "import" };
 import * as JSX from "./jsx";
+import { unique } from "./array";
 
 const aliases = new Map<string, string>();
-for (const lang of BUNDLED_LANGUAGES) {
-    for (const alias of lang.aliases || []) {
-        aliases.set(alias, lang.id);
+let supportedLanguages: string[] = [];
+let supportedThemes: string[] = [];
+
+export async function loadShikiMetadata() {
+    if (aliases.size) return;
+
+    const shiki = await import("shiki");
+    for (const lang of shiki.bundledLanguagesInfo) {
+        for (const alias of lang.aliases || []) {
+            aliases.set(alias, lang.id);
+        }
     }
-}
 
-const supportedLanguages = unique(["text", ...aliases.keys(), ...BUNDLED_LANGUAGES.map((lang) => lang.id)]).sort();
+    supportedLanguages = unique([
+        "text",
+        ...aliases.keys(),
+        ...shiki.bundledLanguagesInfo.map((lang) => lang.id),
+    ]).sort();
+
+    supportedThemes = Object.keys(shiki.bundledThemes);
+}
 
 class DoubleHighlighter {
     private schemes = new Map<string, string>();
 
     constructor(
         private highlighter: Highlighter,
-        private light: Theme,
-        private dark: Theme,
+        private light: BundledTheme,
+        private dark: BundledTheme,
     ) {}
 
     highlight(code: string, lang: string) {
-        const lightTokens = this.highlighter.codeToThemedTokens(code, lang, this.light, { includeExplanation: false });
-        const darkTokens = this.highlighter.codeToThemedTokens(code, lang, this.dark, { includeExplanation: false });
-
-        // If this fails... something went *very* wrong.
-        assert(lightTokens.length === darkTokens.length);
+        const tokens = this.highlighter.codeToTokensWithThemes(code, {
+            themes: { light: this.light, dark: this.dark },
+            lang: lang as BundledLanguage,
+        });
 
         const docEls: JSX.Element[] = [];
 
-        for (const [lightLine, darkLine] of zip(lightTokens, darkTokens)) {
-            // Different themes can have different rules for when colors change... so unfortunately we have to deal with different
-            // sets of tokens.Example: light_plus and dark_plus tokenize " = " differently in the `schemes`
-            // declaration for this file.
-
-            while (lightLine.length && darkLine.length) {
-                // Simple case, same token.
-                if (lightLine[0].content === darkLine[0].content) {
-                    docEls.push(
-                        <span class={this.getClass(lightLine[0].color, darkLine[0].color)}>
-                            {lightLine[0].content}
-                        </span>,
-                    );
-                    lightLine.shift();
-                    darkLine.shift();
-                    continue;
-                }
-
-                if (lightLine[0].content.length < darkLine[0].content.length) {
-                    docEls.push(
-                        <span class={this.getClass(lightLine[0].color, darkLine[0].color)}>
-                            {lightLine[0].content}
-                        </span>,
-                    );
-                    darkLine[0].content = darkLine[0].content.substring(lightLine[0].content.length);
-                    lightLine.shift();
-                    continue;
-                }
-
+        for (const line of tokens) {
+            for (const token of line) {
                 docEls.push(
-                    <span class={this.getClass(lightLine[0].color, darkLine[0].color)}>{darkLine[0].content}</span>,
+                    <span class={this.getClass(token.variants)}>
+                        {token.content}
+                    </span>,
                 );
-                lightLine[0].content = lightLine[0].content.substring(darkLine[0].content.length);
-                darkLine.shift();
             }
 
             docEls.push(<br />);
         }
 
+        docEls.pop(); // Remove last <br>
         docEls.pop(); // Remove last <br>
 
         return JSX.renderElement(<>{docEls}</>);
@@ -121,8 +108,8 @@ class DoubleHighlighter {
         return style.join("\n");
     }
 
-    private getClass(lightColor?: string, darkColor?: string): string {
-        const key = `${lightColor} | ${darkColor}`;
+    private getClass(variants: Record<string, TokenStyles>): string {
+        const key = `${variants["light"].color} | ${variants["dark"].color}`;
         let scheme = this.schemes.get(key);
         if (scheme == null) {
             scheme = `hl-${this.schemes.size}`;
@@ -134,9 +121,11 @@ class DoubleHighlighter {
 
 let highlighter: DoubleHighlighter | undefined;
 
-export async function loadHighlighter(lightTheme: Theme, darkTheme: Theme) {
+export async function loadHighlighter(lightTheme: BundledTheme, darkTheme: BundledTheme) {
     if (highlighter) return;
-    const hl = await getHighlighter({ themes: [lightTheme, darkTheme] });
+
+    const shiki = await import("shiki");
+    const hl = await shiki.getHighlighter({ themes: [lightTheme, darkTheme], langs: getSupportedLanguages() });
     highlighter = new DoubleHighlighter(hl, lightTheme, darkTheme);
 }
 
@@ -145,9 +134,15 @@ export function isSupportedLanguage(lang: string) {
 }
 
 export function getSupportedLanguages(): string[] {
+    ok(supportedLanguages.length > 0, "loadShikiMetadata has not been called");
     return supportedLanguages;
 }
 
+export function getSupportedThemes(): string[] {
+    ok(supportedThemes.length > 0, "loadShikiMetadata has not been called");
+    return supportedThemes;
+}
+
 export function highlight(code: string, lang: string): string {
     assert(highlighter, "Tried to highlight with an uninitialized highlighter");
     if (!isSupportedLanguage(lang)) {

src/lib/utils/options/declaration.ts

@@ -1,4 +1,4 @@
-import type { Theme as ShikiTheme } from "shiki";
+import type { BundledTheme as ShikiTheme } from "shiki" with { "resolution-mode": "import"};
 import type { LogLevel } from "../loggers";
 import type { SortStrategy } from "../sort";
 import { isAbsolute, join, resolve } from "path";

src/lib/utils/options/help.ts

@@ -5,8 +5,7 @@ import {
     ParameterType,
     type DeclarationOption,
 } from "./declaration";
-import { getSupportedLanguages } from "../highlighter";
-import { BUNDLED_THEMES } from "shiki";
+import { getSupportedLanguages, getSupportedThemes } from "../highlighter";
 import type { TranslationProxy } from "../../internationalization/internationalization";
 
 export interface ParameterHelp {
@@ -69,7 +68,7 @@ function toEvenColumns(values: string[], maxLineWidth: number) {
     const columnWidth =
         values.reduce((acc, val) => Math.max(acc, val.length), 0) + 2;
 
-    const numColumns = Math.max(1, Math.min(maxLineWidth / columnWidth));
+    const numColumns = Math.max(1, Math.floor(maxLineWidth / columnWidth));
     let line = "";
     const out: string[] = [];
 
@@ -109,7 +108,7 @@ export function getOptionsHelp(
     output.push(
         "",
         "Supported highlighting themes:",
-        ...toEvenColumns(BUNDLED_THEMES, 80),
+        ...toEvenColumns(getSupportedThemes(), 80),
     );
 
     return output.join("\n");