Skip to content

Commit e9a72d9

Browse files
blackmannbluwyematipicosarah11918
authoredJan 17, 2024
Bump shikiji, use transformers API, expose transformers API (#9643)
* Bump shikiji, use transformers API, expose transformers API * update astro config schema * include shikiji-core * Use default import * address css-variables theme * Remove shikiji markdoc * Improve schema transformers handling * Fix tests * Update changeset * bump shikiji version * Update .changeset/six-scissors-worry.md Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca> * Update wording Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca> --------- Co-authored-by: bluwy <bjornlu.dev@gmail.com> Co-authored-by: Emanuele Stoppa <my.burning@gmail.com> Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>
1 parent 8521ff7 commit e9a72d9

File tree

12 files changed

+132
-105
lines changed

12 files changed

+132
-105
lines changed
 

‎.changeset/polite-dogs-join.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@astrojs/markdoc": patch
3+
---
4+
5+
Removes unnecessary `shikiji` dependency

‎.changeset/six-scissors-worry.md

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
"@astrojs/markdown-remark": minor
3+
"astro": minor
4+
---
5+
6+
Adds a new `markdown.shikiConfig.transformers` config option. You can use this option to transform the Shikiji hast (AST format of the generated HTML) to customize the final HTML. Also updates Shikiji to the latest stable version.
7+
8+
See [Shikiji's documentation](https://shikiji.netlify.app/guide/transformers) for more details about creating your own custom transformers, and [a list of common transformers](https://shikiji.netlify.app/packages/transformers) you can add directly to your project.

‎packages/astro/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@
166166
"resolve": "^1.22.4",
167167
"semver": "^7.5.4",
168168
"server-destroy": "^1.0.1",
169-
"shikiji": "^0.6.13",
169+
"shikiji": "^0.9.18",
170170
"string-width": "^7.0.0",
171171
"strip-ansi": "^7.1.0",
172172
"tsconfck": "^3.0.0",
@@ -224,6 +224,7 @@
224224
"remark-code-titles": "^0.1.2",
225225
"rollup": "^4.5.0",
226226
"sass": "^1.69.5",
227+
"shikiji-core": "^0.9.18",
227228
"srcset-parse": "^1.1.0",
228229
"unified": "^11.0.4"
229230
},

‎packages/astro/src/core/config/schema.ts

+6
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@ import { appendForwardSlash, prependForwardSlash, removeTrailingForwardSlash } f
1717
// This import is required to appease TypeScript!
1818
// See https://github.com/withastro/astro/pull/8762
1919
import 'mdast-util-to-hast';
20+
import 'shikiji-core';
2021

2122
type ShikiLangs = NonNullable<ShikiConfig['langs']>;
2223
type ShikiTheme = NonNullable<ShikiConfig['theme']>;
24+
type ShikiTransformers = NonNullable<ShikiConfig['transformers']>;
2325

2426
const ASTRO_CONFIG_DEFAULTS = {
2527
root: '.',
@@ -275,6 +277,10 @@ export const AstroConfigSchema = z.object({
275277
)
276278
.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.experimentalThemes!),
277279
wrap: z.boolean().or(z.null()).default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.wrap!),
280+
transformers: z
281+
.custom<ShikiTransformers[number]>()
282+
.array()
283+
.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.transformers!),
278284
})
279285
.default({}),
280286
remarkPlugins: z

‎packages/astro/src/core/errors/dev/vite.ts

+8-11
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
1-
import { replaceCssVariables } from '@astrojs/markdown-remark';
21
import * as fs from 'node:fs';
32
import { fileURLToPath } from 'node:url';
4-
import { codeToHtml } from 'shikiji';
3+
import { codeToHtml, createCssVariablesTheme } from 'shikiji';
54
import type { ErrorPayload } from 'vite';
65
import type { ModuleLoader } from '../../module-loader/index.js';
76
import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js';
@@ -124,7 +123,11 @@ export interface AstroErrorPayload {
124123
// Map these to `.js` during error highlighting.
125124
const ALTERNATIVE_JS_EXTS = ['cjs', 'mjs'];
126125
const ALTERNATIVE_MD_EXTS = ['mdoc'];
127-
const INLINE_STYLE_SELECTOR_GLOBAL = /style="(.*?)"/g;
126+
127+
let _cssVariablesTheme: ReturnType<typeof createCssVariablesTheme>;
128+
const cssVariablesTheme = () =>
129+
_cssVariablesTheme ??
130+
(_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' }));
128131

129132
/**
130133
* Generate a payload for Vite's error overlay
@@ -147,21 +150,15 @@ export async function getViteErrorPayload(err: ErrorWithMetadata): Promise<Astro
147150
if (ALTERNATIVE_MD_EXTS.includes(highlighterLang ?? '')) {
148151
highlighterLang = 'md';
149152
}
150-
let highlightedCode = err.fullCode
153+
const highlightedCode = err.fullCode
151154
? await codeToHtml(err.fullCode, {
152155
// @ts-expect-error always assume that shiki can accept the lang string
153156
lang: highlighterLang,
154-
theme: 'css-variables',
157+
theme: cssVariablesTheme(),
155158
lineOptions: err.loc?.line ? [{ line: err.loc.line, classes: ['error-line'] }] : undefined,
156159
})
157160
: undefined;
158161

159-
if (highlightedCode) {
160-
highlightedCode = highlightedCode.replace(INLINE_STYLE_SELECTOR_GLOBAL, (m) =>
161-
replaceCssVariables(m)
162-
);
163-
}
164-
165162
return {
166163
type: 'error',
167164
err: {

‎packages/astro/src/core/errors/overlay.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ const style = /* css */ `
6868
--toggle-border-color: #C3CADB;
6969
7070
/* Syntax Highlighting */
71-
--astro-code-color-text: #000000;
71+
--astro-code-foreground: #000000;
7272
--astro-code-token-constant: #4ca48f;
7373
--astro-code-token-string: #9f722a;
7474
--astro-code-token-comment: #8490b5;
@@ -121,7 +121,7 @@ const style = /* css */ `
121121
--toggle-border-color: #3D4663;
122122
123123
/* Syntax Highlighting */
124-
--astro-code-color-text: #ffffff;
124+
--astro-code-foreground: #ffffff;
125125
--astro-code-token-constant: #90f4e3;
126126
--astro-code-token-string: #f4cf90;
127127
--astro-code-token-comment: #8490b5;

‎packages/integrations/markdoc/package.json

-1
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@
7171
"gray-matter": "^4.0.3",
7272
"htmlparser2": "^9.0.0",
7373
"kleur": "^4.1.5",
74-
"shikiji": "^0.6.13",
7574
"zod": "^3.22.4"
7675
},
7776
"peerDependencies": {

‎packages/markdown/remark/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
"remark-parse": "^11.0.0",
3939
"remark-rehype": "^11.0.0",
4040
"remark-smartypants": "^2.0.0",
41-
"shikiji": "^0.6.13",
41+
"shikiji": "^0.9.18",
4242
"unified": "^11.0.4",
4343
"unist-util-visit": "^5.0.0",
4444
"vfile": "^6.0.1"

‎packages/markdown/remark/src/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export const markdownConfigDefaults: Required<AstroMarkdownOptions> = {
3636
theme: 'github-dark',
3737
experimentalThemes: {},
3838
wrap: false,
39+
transformers: [],
3940
},
4041
remarkPlugins: [],
4142
rehypePlugins: [],

‎packages/markdown/remark/src/shiki.ts

+80-77
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,37 @@
1-
import { bundledLanguages, getHighlighter } from 'shikiji';
1+
import { bundledLanguages, createCssVariablesTheme, getHighlighter } from 'shikiji';
22
import { visit } from 'unist-util-visit';
33
import type { ShikiConfig } from './types.js';
44

55
export interface ShikiHighlighter {
66
highlight(code: string, lang?: string, options?: { inline?: boolean }): string;
77
}
88

9+
// TODO: Remove this special replacement in Astro 5
910
const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
10-
'#000001': 'var(--astro-code-color-text)',
11-
'#000002': 'var(--astro-code-color-background)',
12-
'#000004': 'var(--astro-code-token-constant)',
13-
'#000005': 'var(--astro-code-token-string)',
14-
'#000006': 'var(--astro-code-token-comment)',
15-
'#000007': 'var(--astro-code-token-keyword)',
16-
'#000008': 'var(--astro-code-token-parameter)',
17-
'#000009': 'var(--astro-code-token-function)',
18-
'#000010': 'var(--astro-code-token-string-expression)',
19-
'#000011': 'var(--astro-code-token-punctuation)',
20-
'#000012': 'var(--astro-code-token-link)',
11+
'--astro-code-foreground': '--astro-code-color-text',
12+
'--astro-code-background': '--astro-code-color-background',
2113
};
2214
const COLOR_REPLACEMENT_REGEX = new RegExp(
2315
`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
2416
'g'
2517
);
2618

19+
let _cssVariablesTheme: ReturnType<typeof createCssVariablesTheme>;
20+
const cssVariablesTheme = () =>
21+
_cssVariablesTheme ??
22+
(_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' }));
23+
2724
export async function createShikiHighlighter({
2825
langs = [],
2926
theme = 'github-dark',
3027
experimentalThemes = {},
3128
wrap = false,
29+
transformers = [],
3230
}: ShikiConfig = {}): Promise<ShikiHighlighter> {
3331
const themes = experimentalThemes;
3432

33+
theme = theme === 'css-variables' ? cssVariablesTheme() : theme;
34+
3535
const highlighter = await getHighlighter({
3636
langs: langs.length ? langs : Object.keys(bundledLanguages),
3737
themes: Object.values(themes).length ? Object.values(themes) : [theme],
@@ -53,74 +53,77 @@ export async function createShikiHighlighter({
5353
return highlighter.codeToHtml(code, {
5454
...themeOptions,
5555
lang,
56-
transforms: {
57-
pre(node) {
58-
// Swap to `code` tag if inline
59-
if (inline) {
60-
node.tagName = 'code';
61-
}
62-
63-
// Cast to string as shikiji will always pass them as strings instead of any other types
64-
const classValue = (node.properties.class as string) ?? '';
65-
const styleValue = (node.properties.style as string) ?? '';
66-
67-
// Replace "shiki" class naming with "astro-code"
68-
node.properties.class = classValue.replace(/shiki/g, 'astro-code');
69-
70-
// Handle code wrapping
71-
// if wrap=null, do nothing.
72-
if (wrap === false) {
73-
node.properties.style = styleValue + '; overflow-x: auto;';
74-
} else if (wrap === true) {
75-
node.properties.style =
76-
styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
77-
}
78-
},
79-
line(node) {
80-
// Add "user-select: none;" for "+"/"-" diff symbols.
81-
// Transform `<span class="line"><span style="...">+ something</span></span>
82-
// into `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
83-
if (lang === 'diff') {
84-
const innerSpanNode = node.children[0];
85-
const innerSpanTextNode =
86-
innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
87-
88-
if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
89-
const start = innerSpanTextNode.value[0];
90-
if (start === '+' || start === '-') {
91-
innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
92-
innerSpanNode.children.unshift({
93-
type: 'element',
94-
tagName: 'span',
95-
properties: { style: 'user-select: none;' },
96-
children: [{ type: 'text', value: start }],
97-
});
98-
}
56+
transformers: [
57+
{
58+
pre(node) {
59+
// Swap to `code` tag if inline
60+
if (inline) {
61+
node.tagName = 'code';
9962
}
100-
}
101-
},
102-
code(node) {
103-
if (inline) {
104-
return node.children[0] as typeof node;
105-
}
106-
},
107-
root(node) {
108-
if (Object.values(experimentalThemes).length) {
109-
return;
110-
}
111-
112-
// theme.id for shiki -> shikiji compat
113-
const themeName = typeof theme === 'string' ? theme : theme.name;
114-
if (themeName === 'css-variables') {
115-
// Replace special color tokens to CSS variables
116-
visit(node as any, 'element', (child) => {
117-
if (child.properties?.style) {
118-
child.properties.style = replaceCssVariables(child.properties.style);
63+
64+
// Cast to string as shikiji will always pass them as strings instead of any other types
65+
const classValue = (node.properties.class as string) ?? '';
66+
const styleValue = (node.properties.style as string) ?? '';
67+
68+
// Replace "shiki" class naming with "astro-code"
69+
node.properties.class = classValue.replace(/shiki/g, 'astro-code');
70+
71+
// Handle code wrapping
72+
// if wrap=null, do nothing.
73+
if (wrap === false) {
74+
node.properties.style = styleValue + '; overflow-x: auto;';
75+
} else if (wrap === true) {
76+
node.properties.style =
77+
styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
78+
}
79+
},
80+
line(node) {
81+
// Add "user-select: none;" for "+"/"-" diff symbols.
82+
// Transform `<span class="line"><span style="...">+ something</span></span>
83+
// into `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
84+
if (lang === 'diff') {
85+
const innerSpanNode = node.children[0];
86+
const innerSpanTextNode =
87+
innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
88+
89+
if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
90+
const start = innerSpanTextNode.value[0];
91+
if (start === '+' || start === '-') {
92+
innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
93+
innerSpanNode.children.unshift({
94+
type: 'element',
95+
tagName: 'span',
96+
properties: { style: 'user-select: none;' },
97+
children: [{ type: 'text', value: start }],
98+
});
99+
}
119100
}
120-
});
121-
}
101+
}
102+
},
103+
code(node) {
104+
if (inline) {
105+
return node.children[0] as typeof node;
106+
}
107+
},
108+
root(node) {
109+
if (Object.values(experimentalThemes).length) {
110+
return;
111+
}
112+
113+
// theme.id for shiki -> shikiji compat
114+
const themeName = typeof theme === 'string' ? theme : theme.name;
115+
if (themeName === 'css-variables') {
116+
// Replace special color tokens to CSS variables
117+
visit(node as any, 'element', (child) => {
118+
if (child.properties?.style) {
119+
child.properties.style = replaceCssVariables(child.properties.style);
120+
}
121+
});
122+
}
123+
},
122124
},
123-
},
125+
...transformers,
126+
],
124127
});
125128
},
126129
};

‎packages/markdown/remark/src/types.ts

+6-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { Options as RemarkRehypeOptions } from 'remark-rehype';
44
import type {
55
BuiltinTheme,
66
LanguageRegistration,
7+
ShikijiTransformer,
78
ThemeRegistration,
89
ThemeRegistrationRaw,
910
} from 'shikiji';
@@ -32,11 +33,14 @@ export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlug
3233

3334
export type RemarkRehype = RemarkRehypeOptions;
3435

36+
export type ThemePresets = BuiltinTheme | 'css-variables';
37+
3538
export interface ShikiConfig {
3639
langs?: LanguageRegistration[];
37-
theme?: BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw;
38-
experimentalThemes?: Record<string, BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw>;
40+
theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw;
41+
experimentalThemes?: Record<string, ThemePresets | ThemeRegistration | ThemeRegistrationRaw>;
3942
wrap?: boolean | null;
43+
transformers?: ShikijiTransformer[];
4044
}
4145

4246
export interface AstroMarkdownOptions {

‎pnpm-lock.yaml

+13-10
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)
Please sign in to comment.