Bump shikiji, use transformers API, expose transformers API (#9643)

* Bump shikiji, use transformers API, expose transformers API * update astro config schema * include shikiji-core * Use default import * address css-variables theme * Remove shikiji markdoc * Improve schema transformers handling * Fix tests * Update changeset * bump shikiji version * Update .changeset/six-scissors-worry.md Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca> * Update wording Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca> --------- Co-authored-by: bluwy <bjornlu.dev@gmail.com> Co-authored-by: Emanuele Stoppa <my.burning@gmail.com> Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>
withastro · Jan 17, 2024 · e9a72d9 · e9a72d9
1 parent 8521ff7
commit e9a72d9
Show file tree

Hide file tree

Showing 12 changed files with 132 additions and 105 deletions.
diff --git a/.changeset/polite-dogs-join.md b/.changeset/polite-dogs-join.md
@@ -0,0 +1,5 @@
+---
+"@astrojs/markdoc": patch
+---
+
+Removes unnecessary `shikiji` dependency
diff --git a/.changeset/six-scissors-worry.md b/.changeset/six-scissors-worry.md
@@ -0,0 +1,8 @@
+---
+"@astrojs/markdown-remark": minor
+"astro": minor
+---
+
+Adds a new `markdown.shikiConfig.transformers` config option. You can use this option to transform the Shikiji hast (AST format of the generated HTML) to customize the final HTML. Also updates Shikiji to the latest stable version.
+
+See [Shikiji's documentation](https://shikiji.netlify.app/guide/transformers) for more details about creating your own custom transformers, and [a list of common transformers](https://shikiji.netlify.app/packages/transformers) you can add directly to your project.
diff --git a/packages/astro/package.json b/packages/astro/package.json
@@ -166,7 +166,7 @@
     "resolve": "^1.22.4",
     "semver": "^7.5.4",
     "server-destroy": "^1.0.1",
-    "shikiji": "^0.6.13",
+    "shikiji": "^0.9.18",
     "string-width": "^7.0.0",
     "strip-ansi": "^7.1.0",
     "tsconfck": "^3.0.0",
@@ -224,6 +224,7 @@
     "remark-code-titles": "^0.1.2",
     "rollup": "^4.5.0",
     "sass": "^1.69.5",
+    "shikiji-core": "^0.9.18",
     "srcset-parse": "^1.1.0",
     "unified": "^11.0.4"
   },

diff --git a/packages/astro/src/core/config/schema.ts b/packages/astro/src/core/config/schema.ts
@@ -17,9 +17,11 @@ import { appendForwardSlash, prependForwardSlash, removeTrailingForwardSlash } f
 // This import is required to appease TypeScript!
 // See https://github.com/withastro/astro/pull/8762
 import 'mdast-util-to-hast';
+import 'shikiji-core';
 
 type ShikiLangs = NonNullable<ShikiConfig['langs']>;
 type ShikiTheme = NonNullable<ShikiConfig['theme']>;
+type ShikiTransformers = NonNullable<ShikiConfig['transformers']>;
 
 const ASTRO_CONFIG_DEFAULTS = {
 	root: '.',
@@ -275,6 +277,10 @@ export const AstroConfigSchema = z.object({
 						)
 						.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.experimentalThemes!),
 					wrap: z.boolean().or(z.null()).default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.wrap!),
+					transformers: z
+						.custom<ShikiTransformers[number]>()
+						.array()
+						.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.transformers!),
 				})
 				.default({}),
 			remarkPlugins: z

diff --git a/packages/astro/src/core/errors/dev/vite.ts b/packages/astro/src/core/errors/dev/vite.ts
@@ -1,7 +1,6 @@
-import { replaceCssVariables } from '@astrojs/markdown-remark';
 import * as fs from 'node:fs';
 import { fileURLToPath } from 'node:url';
-import { codeToHtml } from 'shikiji';
+import { codeToHtml, createCssVariablesTheme } from 'shikiji';
 import type { ErrorPayload } from 'vite';
 import type { ModuleLoader } from '../../module-loader/index.js';
 import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js';
@@ -124,7 +123,11 @@ export interface AstroErrorPayload {
 // Map these to `.js` during error highlighting.
 const ALTERNATIVE_JS_EXTS = ['cjs', 'mjs'];
 const ALTERNATIVE_MD_EXTS = ['mdoc'];
-const INLINE_STYLE_SELECTOR_GLOBAL = /style="(.*?)"/g;
+
+let _cssVariablesTheme: ReturnType<typeof createCssVariablesTheme>;
+const cssVariablesTheme = () =>
+	_cssVariablesTheme ??
+	(_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' }));
 
 /**
  * Generate a payload for Vite's error overlay
@@ -147,21 +150,15 @@ export async function getViteErrorPayload(err: ErrorWithMetadata): Promise<Astro
 	if (ALTERNATIVE_MD_EXTS.includes(highlighterLang ?? '')) {
 		highlighterLang = 'md';
 	}
-	let highlightedCode = err.fullCode
+	const highlightedCode = err.fullCode
 		? await codeToHtml(err.fullCode, {
 				// @ts-expect-error always assume that shiki can accept the lang string
 				lang: highlighterLang,
-				theme: 'css-variables',
+				theme: cssVariablesTheme(),
 				lineOptions: err.loc?.line ? [{ line: err.loc.line, classes: ['error-line'] }] : undefined,
 			})
 		: undefined;
 
-	if (highlightedCode) {
-		highlightedCode = highlightedCode.replace(INLINE_STYLE_SELECTOR_GLOBAL, (m) =>
-			replaceCssVariables(m)
-		);
-	}
-
 	return {
 		type: 'error',
 		err: {

diff --git a/packages/astro/src/core/errors/overlay.ts b/packages/astro/src/core/errors/overlay.ts
@@ -68,7 +68,7 @@ const style = /* css */ `
     --toggle-border-color: #C3CADB;
 
   /* Syntax Highlighting */
-  --astro-code-color-text: #000000;
+  --astro-code-foreground: #000000;
   --astro-code-token-constant: #4ca48f;
   --astro-code-token-string: #9f722a;
   --astro-code-token-comment: #8490b5;
@@ -121,7 +121,7 @@ const style = /* css */ `
   --toggle-border-color: #3D4663;
 
   /* Syntax Highlighting */
-  --astro-code-color-text: #ffffff;
+  --astro-code-foreground: #ffffff;
   --astro-code-token-constant: #90f4e3;
   --astro-code-token-string: #f4cf90;
   --astro-code-token-comment: #8490b5;

diff --git a/packages/integrations/markdoc/package.json b/packages/integrations/markdoc/package.json
@@ -71,7 +71,6 @@
     "gray-matter": "^4.0.3",
     "htmlparser2": "^9.0.0",
     "kleur": "^4.1.5",
-    "shikiji": "^0.6.13",
     "zod": "^3.22.4"
   },
   "peerDependencies": {

diff --git a/packages/markdown/remark/package.json b/packages/markdown/remark/package.json
@@ -38,7 +38,7 @@
     "remark-parse": "^11.0.0",
     "remark-rehype": "^11.0.0",
     "remark-smartypants": "^2.0.0",
-    "shikiji": "^0.6.13",
+    "shikiji": "^0.9.18",
     "unified": "^11.0.4",
     "unist-util-visit": "^5.0.0",
     "vfile": "^6.0.1"

diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts
@@ -36,6 +36,7 @@ export const markdownConfigDefaults: Required<AstroMarkdownOptions> = {
 		theme: 'github-dark',
 		experimentalThemes: {},
 		wrap: false,
+		transformers: [],
 	},
 	remarkPlugins: [],
 	rehypePlugins: [],

diff --git a/packages/markdown/remark/src/shiki.ts b/packages/markdown/remark/src/shiki.ts
@@ -1,37 +1,37 @@
-import { bundledLanguages, getHighlighter } from 'shikiji';
+import { bundledLanguages, createCssVariablesTheme, getHighlighter } from 'shikiji';
 import { visit } from 'unist-util-visit';
 import type { ShikiConfig } from './types.js';
 
 export interface ShikiHighlighter {
 	highlight(code: string, lang?: string, options?: { inline?: boolean }): string;
 }
 
+// TODO: Remove this special replacement in Astro 5
 const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
-	'#000001': 'var(--astro-code-color-text)',
-	'#000002': 'var(--astro-code-color-background)',
-	'#000004': 'var(--astro-code-token-constant)',
-	'#000005': 'var(--astro-code-token-string)',
-	'#000006': 'var(--astro-code-token-comment)',
-	'#000007': 'var(--astro-code-token-keyword)',
-	'#000008': 'var(--astro-code-token-parameter)',
-	'#000009': 'var(--astro-code-token-function)',
-	'#000010': 'var(--astro-code-token-string-expression)',
-	'#000011': 'var(--astro-code-token-punctuation)',
-	'#000012': 'var(--astro-code-token-link)',
+	'--astro-code-foreground': '--astro-code-color-text',
+	'--astro-code-background': '--astro-code-color-background',
 };
 const COLOR_REPLACEMENT_REGEX = new RegExp(
 	`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
 	'g'
 );
 
+let _cssVariablesTheme: ReturnType<typeof createCssVariablesTheme>;
+const cssVariablesTheme = () =>
+	_cssVariablesTheme ??
+	(_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' }));
+
 export async function createShikiHighlighter({
 	langs = [],
 	theme = 'github-dark',
 	experimentalThemes = {},
 	wrap = false,
+	transformers = [],
 }: ShikiConfig = {}): Promise<ShikiHighlighter> {
 	const themes = experimentalThemes;
 
+	theme = theme === 'css-variables' ? cssVariablesTheme() : theme;
+
 	const highlighter = await getHighlighter({
 		langs: langs.length ? langs : Object.keys(bundledLanguages),
 		themes: Object.values(themes).length ? Object.values(themes) : [theme],
@@ -53,74 +53,77 @@ export async function createShikiHighlighter({
 			return highlighter.codeToHtml(code, {
 				...themeOptions,
 				lang,
-				transforms: {
-					pre(node) {
-						// Swap to `code` tag if inline
-						if (inline) {
-							node.tagName = 'code';
-						}
-
-						// Cast to string as shikiji will always pass them as strings instead of any other types
-						const classValue = (node.properties.class as string) ?? '';
-						const styleValue = (node.properties.style as string) ?? '';
-
-						// Replace "shiki" class naming with "astro-code"
-						node.properties.class = classValue.replace(/shiki/g, 'astro-code');
-
-						// Handle code wrapping
-						// if wrap=null, do nothing.
-						if (wrap === false) {
-							node.properties.style = styleValue + '; overflow-x: auto;';
-						} else if (wrap === true) {
-							node.properties.style =
-								styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
-						}
-					},
-					line(node) {
-						// Add "user-select: none;" for "+"/"-" diff symbols.
-						// Transform `<span class="line"><span style="...">+ something</span></span>
-						// into      `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
-						if (lang === 'diff') {
-							const innerSpanNode = node.children[0];
-							const innerSpanTextNode =
-								innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
-
-							if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
-								const start = innerSpanTextNode.value[0];
-								if (start === '+' || start === '-') {
-									innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
-									innerSpanNode.children.unshift({
-										type: 'element',
-										tagName: 'span',
-										properties: { style: 'user-select: none;' },
-										children: [{ type: 'text', value: start }],
-									});
-								}
+				transformers: [
+					{
+						pre(node) {
+							// Swap to `code` tag if inline
+							if (inline) {
+								node.tagName = 'code';
 							}
-						}
-					},
-					code(node) {
-						if (inline) {
-							return node.children[0] as typeof node;
-						}
-					},
-					root(node) {
-						if (Object.values(experimentalThemes).length) {
-							return;
-						}
-
-						// theme.id for shiki -> shikiji compat
-						const themeName = typeof theme === 'string' ? theme : theme.name;
-						if (themeName === 'css-variables') {
-							// Replace special color tokens to CSS variables
-							visit(node as any, 'element', (child) => {
-								if (child.properties?.style) {
-									child.properties.style = replaceCssVariables(child.properties.style);
+
+							// Cast to string as shikiji will always pass them as strings instead of any other types
+							const classValue = (node.properties.class as string) ?? '';
+							const styleValue = (node.properties.style as string) ?? '';
+
+							// Replace "shiki" class naming with "astro-code"
+							node.properties.class = classValue.replace(/shiki/g, 'astro-code');
+
+							// Handle code wrapping
+							// if wrap=null, do nothing.
+							if (wrap === false) {
+								node.properties.style = styleValue + '; overflow-x: auto;';
+							} else if (wrap === true) {
+								node.properties.style =
+									styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
+							}
+						},
+						line(node) {
+							// Add "user-select: none;" for "+"/"-" diff symbols.
+							// Transform `<span class="line"><span style="...">+ something</span></span>
+							// into      `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
+							if (lang === 'diff') {
+								const innerSpanNode = node.children[0];
+								const innerSpanTextNode =
+									innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
+
+								if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
+									const start = innerSpanTextNode.value[0];
+									if (start === '+' || start === '-') {
+										innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
+										innerSpanNode.children.unshift({
+											type: 'element',
+											tagName: 'span',
+											properties: { style: 'user-select: none;' },
+											children: [{ type: 'text', value: start }],
+										});
+									}
 								}
-							});
-						}
+							}
+						},
+						code(node) {
+							if (inline) {
+								return node.children[0] as typeof node;
+							}
+						},
+						root(node) {
+							if (Object.values(experimentalThemes).length) {
+								return;
+							}
+
+							// theme.id for shiki -> shikiji compat
+							const themeName = typeof theme === 'string' ? theme : theme.name;
+							if (themeName === 'css-variables') {
+								// Replace special color tokens to CSS variables
+								visit(node as any, 'element', (child) => {
+									if (child.properties?.style) {
+										child.properties.style = replaceCssVariables(child.properties.style);
+									}
+								});
+							}
+						},
 					},
-				},
+					...transformers,
+				],
 			});
 		},
 	};

diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts
@@ -4,6 +4,7 @@ import type { Options as RemarkRehypeOptions } from 'remark-rehype';
 import type {
 	BuiltinTheme,
 	LanguageRegistration,
+	ShikijiTransformer,
 	ThemeRegistration,
 	ThemeRegistrationRaw,
 } from 'shikiji';
@@ -32,11 +33,14 @@ export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlug
 
 export type RemarkRehype = RemarkRehypeOptions;
 
+export type ThemePresets = BuiltinTheme | 'css-variables';
+
 export interface ShikiConfig {
 	langs?: LanguageRegistration[];
-	theme?: BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw;
-	experimentalThemes?: Record<string, BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw>;
+	theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw;
+	experimentalThemes?: Record<string, ThemePresets | ThemeRegistration | ThemeRegistrationRaw>;
 	wrap?: boolean | null;
+	transformers?: ShikijiTransformer[];
 }
 
 export interface AstroMarkdownOptions {