Skip to content

Commit

Permalink
Refactor shikiji syntax highlighting code (#9083)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bluwy
bluwy committed Nov 14, 2023
1 parent 6f5de8d commit 4537ecf
Show file tree
Hide file tree
Showing 10 changed files with 198 additions and 293 deletions.
5 changes: 5 additions & 0 deletions .changeset/fast-zebras-divide.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@astrojs/markdown-remark': minor
---

Exports `createShikiHighlighter` for low-level syntax highlighting usage
6 changes: 6 additions & 0 deletions .changeset/mighty-zebras-clap.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@astrojs/markdoc': patch
'astro': patch
---

Uses new `createShikiHighlighter` API from `@astrojs/markdown-remark` to avoid code duplication
60 changes: 6 additions & 54 deletions packages/astro/components/Code.astro
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@ import type {
ThemeRegistration,
ThemeRegistrationRaw,
} from 'shikiji';
import { visit } from 'unist-util-visit';
import { getCachedHighlighter, replaceCssVariables } from '../dist/core/shiki.js';
import { getCachedHighlighter } from '../dist/core/shiki.js';
interface Props {
/** The code to highlight. Required. */
Expand Down Expand Up @@ -94,60 +93,13 @@ if (typeof lang === 'object') {
const highlighter = await getCachedHighlighter({
langs: [lang],
themes: Object.values(experimentalThemes).length ? Object.values(experimentalThemes) : [theme],
theme,
experimentalThemes,
wrap,
});
const themeOptions = Object.values(experimentalThemes).length
? { themes: experimentalThemes }
: { theme };
const html = highlighter.codeToHtml(code, {
lang: typeof lang === 'string' ? lang : lang.name,
...themeOptions,
transforms: {
pre(node) {
// Swap to `code` tag if inline
if (inline) {
node.tagName = 'code';
}
// Cast to string as shikiji will always pass them as strings instead of any other types
const classValue = (node.properties.class as string) ?? '';
const styleValue = (node.properties.style as string) ?? '';
// Replace "shiki" class naming with "astro-code"
node.properties.class = classValue.replace(/shiki/g, 'astro-code');
// Handle code wrapping
// if wrap=null, do nothing.
if (wrap === false) {
node.properties.style = styleValue + '; overflow-x: auto;';
} else if (wrap === true) {
node.properties.style =
styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
}
},
code(node) {
if (inline) {
return node.children[0] as typeof node;
}
},
root(node) {
if (Object.values(experimentalThemes).length) {
return;
}
// theme.id for shiki -> shikiji compat
const themeName = typeof theme === 'string' ? theme : theme.name;
if (themeName === 'css-variables') {
// Replace special color tokens to CSS variables
visit(node as any, 'element', (child) => {
if (child.properties?.style) {
child.properties.style = replaceCssVariables(child.properties.style);
}
});
}
},
},
const html = highlighter.highlight(code, typeof lang === 'string' ? lang : lang.name, {
inline,
});
---

Expand Down
2 changes: 1 addition & 1 deletion packages/astro/src/core/errors/dev/vite.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { codeToHtml } from 'shikiji';
import type { ErrorPayload } from 'vite';
import { replaceCssVariables } from '@astrojs/markdown-remark';
import type { ModuleLoader } from '../../module-loader/index.js';
import { replaceCssVariables } from '../../shiki.js';
import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js';
import { AstroError, type ErrorWithMetadata } from '../errors.js';
import { createSafeError } from '../utils.js';
Expand Down
39 changes: 8 additions & 31 deletions packages/astro/src/core/shiki.ts
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,13 @@
import { getHighlighter, type Highlighter } from 'shikiji';
import {
createShikiHighlighter,
type ShikiHighlighter,
type ShikiConfig,
} from '@astrojs/markdown-remark';

type HighlighterOptions = NonNullable<Parameters<typeof getHighlighter>[0]>;

const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
'#000001': 'var(--astro-code-color-text)',
'#000002': 'var(--astro-code-color-background)',
'#000004': 'var(--astro-code-token-constant)',
'#000005': 'var(--astro-code-token-string)',
'#000006': 'var(--astro-code-token-comment)',
'#000007': 'var(--astro-code-token-keyword)',
'#000008': 'var(--astro-code-token-parameter)',
'#000009': 'var(--astro-code-token-function)',
'#000010': 'var(--astro-code-token-string-expression)',
'#000011': 'var(--astro-code-token-punctuation)',
'#000012': 'var(--astro-code-token-link)',
};
const COLOR_REPLACEMENT_REGEX = new RegExp(
`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
'g'
);

// Caches Promise<Highlighter> for reuse when the same theme and langs are provided
// Caches Promise<ShikiHighlighter> for reuse when the same theme and langs are provided
const cachedHighlighters = new Map();

/**
* shiki -> shikiji compat as we need to manually replace it
*/
export function replaceCssVariables(str: string) {
return str.replace(COLOR_REPLACEMENT_REGEX, (match) => ASTRO_COLOR_REPLACEMENTS[match] || match);
}

export function getCachedHighlighter(opts: HighlighterOptions): Promise<Highlighter> {
export function getCachedHighlighter(opts: ShikiConfig): Promise<ShikiHighlighter> {
// Always sort keys before stringifying to make sure objects match regardless of parameter ordering
const key = JSON.stringify(opts, Object.keys(opts).sort());

Expand All @@ -39,7 +16,7 @@ export function getCachedHighlighter(opts: HighlighterOptions): Promise<Highligh
return cachedHighlighters.get(key);
}

const highlighter = getHighlighter(opts);
const highlighter = createShikiHighlighter(opts);
cachedHighlighters.set(key, highlighter);

return highlighter;
Expand Down
105 changes: 5 additions & 100 deletions packages/integrations/markdoc/src/extensions/shiki.ts
View file
Original file line number Diff line number Diff line change
@@ -1,107 +1,19 @@
import Markdoc from '@markdoc/markdoc';
import { createShikiHighlighter } from '@astrojs/markdown-remark';
import type { ShikiConfig } from 'astro';
import { unescapeHTML } from 'astro/runtime/server/index.js';
import { bundledLanguages, getHighlighter, type Highlighter } from 'shikiji';
import type { AstroMarkdocConfig } from '../config.js';

const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
'#000001': 'var(--astro-code-color-text)',
'#000002': 'var(--astro-code-color-background)',
'#000004': 'var(--astro-code-token-constant)',
'#000005': 'var(--astro-code-token-string)',
'#000006': 'var(--astro-code-token-comment)',
'#000007': 'var(--astro-code-token-keyword)',
'#000008': 'var(--astro-code-token-parameter)',
'#000009': 'var(--astro-code-token-function)',
'#000010': 'var(--astro-code-token-string-expression)',
'#000011': 'var(--astro-code-token-punctuation)',
'#000012': 'var(--astro-code-token-link)',
};
const COLOR_REPLACEMENT_REGEX = new RegExp(
`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
'g'
);

const PRE_SELECTOR = /<pre class="(.*?)shiki(.*?)"/;
const LINE_SELECTOR = /<span class="line"><span style="(.*?)">([\+|\-])/g;
const INLINE_STYLE_SELECTOR = /style="(.*?)"/;
const INLINE_STYLE_SELECTOR_GLOBAL = /style="(.*?)"/g;

/**
* Note: cache only needed for dev server reloads, internal test suites, and manual calls to `Markdoc.transform` by the user.
* Otherwise, `shiki()` is only called once per build, NOT once per page, so a cache isn't needed!
*/
const highlighterCache = new Map<string, Highlighter>();

export default async function shiki({
langs = [],
theme = 'github-dark',
wrap = false,
}: ShikiConfig = {}): Promise<AstroMarkdocConfig> {
const cacheId = typeof theme === 'string' ? theme : theme.name || '';
let highlighter = highlighterCache.get(cacheId)!;
if (!highlighter) {
highlighter = await getHighlighter({
langs: langs.length ? langs : Object.keys(bundledLanguages),
themes: [theme],
});
highlighterCache.set(cacheId, highlighter);
}
export default async function shiki(config?: ShikiConfig): Promise<AstroMarkdocConfig> {
const highlighter = await createShikiHighlighter(config);

return {
nodes: {
fence: {
attributes: Markdoc.nodes.fence.attributes!,
transform({ attributes }) {
let lang: string;

if (typeof attributes.language === 'string') {
const langExists = highlighter
.getLoadedLanguages()
.includes(attributes.language as any);
if (langExists) {
lang = attributes.language;
} else {
console.warn(
`[Shiki highlighter] The language "${attributes.language}" doesn't exist, falling back to plaintext.`
);
lang = 'plaintext';
}
} else {
lang = 'plaintext';
}

let html = highlighter.codeToHtml(attributes.content, { lang, theme });

// Q: Could these regexes match on a user's inputted code blocks?
// A: Nope! All rendered HTML is properly escaped.
// Ex. If a user typed `<span class="line"` into a code block,
// It would become this before hitting our regexes:
// &lt;span class=&quot;line&quot;

html = html.replace(PRE_SELECTOR, `<pre class="$1astro-code$2"`);
// Add "user-select: none;" for "+"/"-" diff symbols
if (attributes.language === 'diff') {
html = html.replace(
LINE_SELECTOR,
'<span class="line"><span style="$1"><span style="user-select: none;">$2</span>'
);
}

if (wrap === false) {
html = html.replace(INLINE_STYLE_SELECTOR, 'style="$1; overflow-x: auto;"');
} else if (wrap === true) {
html = html.replace(
INLINE_STYLE_SELECTOR,
'style="$1; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;"'
);
}

// theme.id for shiki -> shikiji compat
const themeName = typeof theme === 'string' ? theme : theme.name;
if (themeName === 'css-variables') {
html = html.replace(INLINE_STYLE_SELECTOR_GLOBAL, (m) => replaceCssVariables(m));
}
const lang = typeof attributes.language === 'string' ? attributes.language : 'plaintext';
const html = highlighter.highlight(attributes.content, lang);

// Use `unescapeHTML` to return `HTMLString` for Astro renderer to inline as HTML
return unescapeHTML(html) as any;
Expand All @@ -110,10 +22,3 @@ export default async function shiki({
},
};
}

/**
* shiki -> shikiji compat as we need to manually replace it
*/
function replaceCssVariables(str: string) {
return str.replace(COLOR_REPLACEMENT_REGEX, (match) => ASTRO_COLOR_REPLACEMENTS[match] || match);
}
1 change: 1 addition & 0 deletions packages/markdown/remark/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export { rehypeHeadingIds } from './rehype-collect-headings.js';
export { remarkCollectImages } from './remark-collect-images.js';
export { remarkPrism } from './remark-prism.js';
export { remarkShiki } from './remark-shiki.js';
export { createShikiHighlighter, replaceCssVariables, type ShikiHighlighter } from './shiki.js';
export * from './types.js';

export const markdownConfigDefaults: Omit<Required<AstroMarkdownOptions>, 'drafts'> = {
Expand Down

0 comments on commit 4537ecf

Please sign in to comment.