Refactor shikiji syntax highlighting code (#9083)

3 files changed, 143 insertions, 106 deletions

diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts
index 61f97072b..a60ab88c0 100644
--- a/packages/markdown/remark/src/index.ts
+++ b/packages/markdown/remark/src/index.ts
@@ -32,6 +32,7 @@ export { rehypeHeadingIds } from './rehype-collect-headings.js';
 export { remarkCollectImages } from './remark-collect-images.js';
 export { remarkPrism } from './remark-prism.js';
 export { remarkShiki } from './remark-shiki.js';
+export { createShikiHighlighter, replaceCssVariables, type ShikiHighlighter } from './shiki.js';
 export * from './types.js';
 
 export const markdownConfigDefaults: Omit<Required<AstroMarkdownOptions>, 'drafts'> = {
diff --git a/packages/markdown/remark/src/remark-shiki.ts b/packages/markdown/remark/src/remark-shiki.ts
index 4eaae5ff2..8c6e93242 100644
--- a/packages/markdown/remark/src/remark-shiki.ts
+++ b/packages/markdown/remark/src/remark-shiki.ts
@@ -1,109 +1,17 @@
-import { bundledLanguages, getHighlighter, type Highlighter } from 'shikiji';
 import { visit } from 'unist-util-visit';
 import type { RemarkPlugin, ShikiConfig } from './types.js';
+import { createShikiHighlighter, type ShikiHighlighter } from './shiki.js';
 
-const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
-	'#000001': 'var(--astro-code-color-text)',
-	'#000002': 'var(--astro-code-color-background)',
-	'#000004': 'var(--astro-code-token-constant)',
-	'#000005': 'var(--astro-code-token-string)',
-	'#000006': 'var(--astro-code-token-comment)',
-	'#000007': 'var(--astro-code-token-keyword)',
-	'#000008': 'var(--astro-code-token-parameter)',
-	'#000009': 'var(--astro-code-token-function)',
-	'#000010': 'var(--astro-code-token-string-expression)',
-	'#000011': 'var(--astro-code-token-punctuation)',
-	'#000012': 'var(--astro-code-token-link)',
-};
-const COLOR_REPLACEMENT_REGEX = new RegExp(
-	`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
-	'g'
-);
-
-/**
- * getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page,
- * cache it here as much as possible. Make sure that your highlighters can be cached, state-free.
- * We make this async, so that multiple calls to parse markdown still share the same highlighter.
- */
-const highlighterCacheAsync = new Map<string, Promise<Highlighter>>();
-
-export function remarkShiki({
-	langs = [],
-	theme = 'github-dark',
-	experimentalThemes = {},
-	wrap = false,
-}: ShikiConfig = {}): ReturnType<RemarkPlugin> {
-	const themes = experimentalThemes;
-
-	const cacheId =
-		Object.values(themes)
-			.map((t) => (typeof t === 'string' ? t : t.name ?? ''))
-			.join(',') +
-		(typeof theme === 'string' ? theme : theme.name ?? '') +
-		langs.map((l) => l.name ?? (l as any).id).join(',');
-
-	let highlighterAsync = highlighterCacheAsync.get(cacheId);
-	if (!highlighterAsync) {
-		highlighterAsync = getHighlighter({
-			langs: langs.length ? langs : Object.keys(bundledLanguages),
-			themes: Object.values(themes).length ? Object.values(themes) : [theme],
-		});
-		highlighterCacheAsync.set(cacheId, highlighterAsync);
-	}
+export function remarkShiki(config?: ShikiConfig): ReturnType<RemarkPlugin> {
+	let highlighterAsync: Promise<ShikiHighlighter> | undefined;
 
 	return async (tree: any) => {
-		const highlighter = await highlighterAsync!;
+		highlighterAsync ??= createShikiHighlighter(config);
+		const highlighter = await highlighterAsync;
 
 		visit(tree, 'code', (node) => {
-			let lang: string;
-
-			if (typeof node.lang === 'string') {
-				const langExists = highlighter.getLoadedLanguages().includes(node.lang);
-				if (langExists) {
-					lang = node.lang;
-				} else {
-					// eslint-disable-next-line no-console
-					console.warn(`The language "${node.lang}" doesn't exist, falling back to plaintext.`);
-					lang = 'plaintext';
-				}
-			} else {
-				lang = 'plaintext';
-			}
-
-			let themeOptions = Object.values(themes).length ? { themes } : { theme };
-			let html = highlighter.codeToHtml(node.value, { ...themeOptions, lang });
-
-			// Q: Couldn't these regexes match on a user's inputted code blocks?
-			// A: Nope! All rendered HTML is properly escaped.
-			// Ex. If a user typed `<span class="line"` into a code block,
-			// It would become this before hitting our regexes:
-			// &lt;span class=&quot;line&quot;
-
-			// Replace "shiki" class naming with "astro".
-			html = html.replace(/<pre class="(.*?)shiki(.*?)"/, `<pre class="$1astro-code$2"`);
-			// Add "user-select: none;" for "+"/"-" diff symbols
-			if (node.lang === 'diff') {
-				html = html.replace(
-					/<span class="line"><span style="(.*?)">([\+|\-])/g,
-					'<span class="line"><span style="$1"><span style="user-select: none;">$2</span>'
-				);
-			}
-			// Handle code wrapping
-			// if wrap=null, do nothing.
-			if (wrap === false) {
-				html = html.replace(/style="(.*?)"/, 'style="$1; overflow-x: auto;"');
-			} else if (wrap === true) {
-				html = html.replace(
-					/style="(.*?)"/,
-					'style="$1; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;"'
-				);
-			}
-
-			// theme.id for shiki -> shikiji compat
-			const themeName = typeof theme === 'string' ? theme : theme.name;
-			if (themeName === 'css-variables') {
-				html = html.replace(/style="(.*?)"/g, (m) => replaceCssVariables(m));
-			}
+			const lang = typeof node.lang === 'string' ? node.lang : 'plaintext';
+			const html = highlighter.highlight(node.value, lang);
 
 			node.type = 'html';
 			node.value = html;
@@ -111,10 +19,3 @@ export function remarkShiki({
 		});
 	};
 }
-
-/**
- * shiki -> shikiji compat as we need to manually replace it
- */
-function replaceCssVariables(str: string) {
-	return str.replace(COLOR_REPLACEMENT_REGEX, (match) => ASTRO_COLOR_REPLACEMENTS[match] || match);
-}
diff --git a/packages/markdown/remark/src/shiki.ts b/packages/markdown/remark/src/shiki.ts
new file mode 100644
index 000000000..477ab2184
--- /dev/null
+++ b/packages/markdown/remark/src/shiki.ts
@@ -0,0 +1,135 @@
+import { bundledLanguages, getHighlighter } from 'shikiji';
+import { visit } from 'unist-util-visit';
+import type { ShikiConfig } from './types.js';
+
+export interface ShikiHighlighter {
+	highlight(code: string, lang?: string, options?: { inline?: boolean }): string;
+}
+
+const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
+	'#000001': 'var(--astro-code-color-text)',
+	'#000002': 'var(--astro-code-color-background)',
+	'#000004': 'var(--astro-code-token-constant)',
+	'#000005': 'var(--astro-code-token-string)',
+	'#000006': 'var(--astro-code-token-comment)',
+	'#000007': 'var(--astro-code-token-keyword)',
+	'#000008': 'var(--astro-code-token-parameter)',
+	'#000009': 'var(--astro-code-token-function)',
+	'#000010': 'var(--astro-code-token-string-expression)',
+	'#000011': 'var(--astro-code-token-punctuation)',
+	'#000012': 'var(--astro-code-token-link)',
+};
+const COLOR_REPLACEMENT_REGEX = new RegExp(
+	`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
+	'g'
+);
+
+export async function createShikiHighlighter({
+	langs = [],
+	theme = 'github-dark',
+	experimentalThemes = {},
+	wrap = false,
+}: ShikiConfig = {}): Promise<ShikiHighlighter> {
+	const themes = experimentalThemes;
+
+	const highlighter = await getHighlighter({
+		langs: langs.length ? langs : Object.keys(bundledLanguages),
+		themes: Object.values(themes).length ? Object.values(themes) : [theme],
+	});
+
+	const loadedLanguages = highlighter.getLoadedLanguages();
+
+	return {
+		highlight(code, lang = 'plaintext', options) {
+			if (lang !== 'plaintext' && !loadedLanguages.includes(lang)) {
+				// eslint-disable-next-line no-console
+				console.warn(`[Shiki] The language "${lang}" doesn't exist, falling back to "plaintext".`);
+				lang = 'plaintext';
+			}
+
+			const themeOptions = Object.values(themes).length ? { themes } : { theme };
+			const inline = options?.inline ?? false;
+
+			return highlighter.codeToHtml(code, {
+				...themeOptions,
+				lang,
+				transforms: {
+					pre(node) {
+						// Swap to `code` tag if inline
+						if (inline) {
+							node.tagName = 'code';
+						}
+
+						// Cast to string as shikiji will always pass them as strings instead of any other types
+						const classValue = (node.properties.class as string) ?? '';
+						const styleValue = (node.properties.style as string) ?? '';
+
+						// Replace "shiki" class naming with "astro-code"
+						node.properties.class = classValue.replace(/shiki/g, 'astro-code');
+
+						// Handle code wrapping
+						// if wrap=null, do nothing.
+						if (wrap === false) {
+							node.properties.style = styleValue + '; overflow-x: auto;';
+						} else if (wrap === true) {
+							node.properties.style =
+								styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
+						}
+					},
+					line(node) {
+						// Add "user-select: none;" for "+"/"-" diff symbols.
+						// Transform `<span class="line"><span style="...">+ something</span></span>
+						// into      `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
+						if (lang === 'diff') {
+							const innerSpanNode = node.children[0];
+							const innerSpanTextNode =
+								innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
+
+							if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
+								const start = innerSpanTextNode.value[0];
+								if (start === '+' || start === '-') {
+									innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
+									innerSpanNode.children.unshift({
+										type: 'element',
+										tagName: 'span',
+										properties: { style: 'user-select: none;' },
+										children: [{ type: 'text', value: start }],
+									});
+								}
+							}
+						}
+					},
+					code(node) {
+						if (inline) {
+							return node.children[0] as typeof node;
+						}
+					},
+					root(node) {
+						if (Object.values(experimentalThemes).length) {
+							return;
+						}
+
+						// theme.id for shiki -> shikiji compat
+						const themeName = typeof theme === 'string' ? theme : theme.name;
+						if (themeName === 'css-variables') {
+							// Replace special color tokens to CSS variables
+							visit(node as any, 'element', (child) => {
+								if (child.properties?.style) {
+									child.properties.style = replaceCssVariables(child.properties.style);
+								}
+							});
+						}
+					},
+				},
+			});
+		},
+	};
+}
+
+/**
+ * shiki -> shikiji compat as we need to manually replace it
+ * @internal Exported for error overlay use only
+ */
+export function replaceCssVariables(str: string) {
+	return str.replace(COLOR_REPLACEMENT_REGEX, (match) => ASTRO_COLOR_REPLACEMENTS[match] || match);
+}