Improve Markdown + Components usage (#3410)

* feat: use internal MDX tooling for markdown + components

* fix: improve MD + component tests

* chore: add changeset

* fix: make tsc happy

* fix(#3319): add regression test for component children

* fix(markdown): support HTML comments in markdown

* fix(#2474): ensure namespaced components are properly handled in markdown pages

* fix(#3220): ensure html in markdown pages does not have extra surrounding space

* fix(#3264): ensure that remark files pass in file information

* fix(#3254): enable experimentalStaticExtraction for `.md` pages

* fix: revert parsing change

* fix: remove `markdown.mode` option

11 files changed, 211 insertions, 88 deletions

diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts
index bf660a508..d942bf7bf 100644
--- a/packages/markdown/remark/src/index.ts
+++ b/packages/markdown/remark/src/index.ts
@@ -2,10 +2,10 @@ import type { MarkdownRenderingOptions, MarkdownRenderingResult } from './types'
 
 import createCollectHeaders from './rehype-collect-headers.js';
 import scopedStyles from './remark-scoped-styles.js';
-import { remarkExpressions, loadRemarkExpressions } from './remark-expressions.js';
 import rehypeExpressions from './rehype-expressions.js';
 import rehypeIslands from './rehype-islands.js';
-import { remarkJsx, loadRemarkJsx } from './remark-jsx.js';
+import remarkMdxish from './remark-mdxish.js';
+import remarkMarkAndUnravel from './remark-mark-and-unravel.js';
 import rehypeJsx from './rehype-jsx.js';
 import rehypeEscape from './rehype-escape.js';
 import remarkPrism from './remark-prism.js';
@@ -18,27 +18,33 @@ import markdown from 'remark-parse';
 import markdownToHtml from 'remark-rehype';
 import rehypeStringify from 'rehype-stringify';
 import rehypeRaw from 'rehype-raw';
+import Slugger from 'github-slugger';
+import { VFile } from 'vfile';
 
 export * from './types.js';
 
 export const DEFAULT_REMARK_PLUGINS = ['remark-gfm', 'remark-smartypants'];
 export const DEFAULT_REHYPE_PLUGINS = [];
 
+const slugger = new Slugger();
+export function slug(value: string): string {
+	return slugger.slug(value);
+}
+
 /** Shared utility for rendering markdown */
 export async function renderMarkdown(
 	content: string,
-	opts: MarkdownRenderingOptions
+	opts: MarkdownRenderingOptions = {}
 ): Promise<MarkdownRenderingResult> {
-	let { mode, syntaxHighlight, shikiConfig, remarkPlugins, rehypePlugins } = opts;
+	let { fileURL, mode = 'mdx', syntaxHighlight = 'shiki', shikiConfig = {}, remarkPlugins = [], rehypePlugins = [] } = opts;
+	const input = new VFile({ value: content, path: fileURL })
 	const scopedClassName = opts.$?.scopedClassName;
 	const isMDX = mode === 'mdx';
 	const { headers, rehypeCollectHeaders } = createCollectHeaders();
 
-	await Promise.all([loadRemarkExpressions(), loadRemarkJsx()]); // Vite bug: dynamically import() these because of CJS interop (this will cache)
-
 	let parser = unified()
 		.use(markdown)
-		.use(isMDX ? [remarkJsx, remarkExpressions] : [])
+		.use(isMDX ? [remarkMdxish, remarkMarkAndUnravel] : [])
 		.use([remarkUnwrap]);
 
 	if (remarkPlugins.length === 0 && rehypePlugins.length === 0) {
@@ -68,7 +74,13 @@ export async function renderMarkdown(
 			markdownToHtml as any,
 			{
 				allowDangerousHtml: true,
-				passThrough: ['raw', 'mdxTextExpression', 'mdxJsxTextElement', 'mdxJsxFlowElement'],
+				passThrough: [
+					'raw',
+					'mdxFlowExpression',
+					'mdxJsxFlowElement',
+					'mdxJsxTextElement',
+					'mdxTextExpression',
+				],
 			},
 		],
 	]);
@@ -87,7 +99,7 @@ export async function renderMarkdown(
 		const vfile = await parser
 			.use([rehypeCollectHeaders])
 			.use(rehypeStringify, { allowDangerousHtml: true })
-			.process(content);
+			.process(input);
 		result = vfile.toString();
 	} catch (err) {
 		console.error(err);
diff --git a/packages/markdown/remark/src/mdast-util-mdxish.ts b/packages/markdown/remark/src/mdast-util-mdxish.ts
new file mode 100644
index 000000000..52a99deeb
--- /dev/null
+++ b/packages/markdown/remark/src/mdast-util-mdxish.ts
@@ -0,0 +1,18 @@
+import {
+  mdxExpressionFromMarkdown,
+  mdxExpressionToMarkdown
+} from 'mdast-util-mdx-expression'
+import {mdxJsxFromMarkdown, mdxJsxToMarkdown} from 'mdast-util-mdx-jsx'
+
+export function mdxFromMarkdown(): any {
+  return [mdxExpressionFromMarkdown, mdxJsxFromMarkdown]
+}
+
+export function mdxToMarkdown(): any {
+  return {
+    extensions: [
+      mdxExpressionToMarkdown,
+      mdxJsxToMarkdown,
+    ]
+  }
+}
diff --git a/packages/markdown/remark/src/rehype-collect-headers.ts b/packages/markdown/remark/src/rehype-collect-headers.ts
index 927f96590..77126ab7e 100644
--- a/packages/markdown/remark/src/rehype-collect-headers.ts
+++ b/packages/markdown/remark/src/rehype-collect-headers.ts
@@ -17,15 +17,38 @@ export default function createCollectHeaders() {
 				if (!level) return;
 				const depth = Number.parseInt(level);
 
+				let raw = '';
 				let text = '';
-
-				visit(node, 'text', (child) => {
-					text += child.value;
+				let isJSX = false;
+				visit(node, (child) => {
+					if (child.type === 'element') {
+						return;
+					}
+					if (child.type === 'raw') {
+						// HACK: serialized JSX from internal plugins, ignore these for slug
+						if (child.value.startsWith('\n<') || child.value.endsWith('>\n')) {
+							raw += child.value.replace(/^\n|\n$/g, '');
+							return;
+						}
+					}
+					if (child.type === 'text' || child.type === 'raw')  {
+						raw += child.value;
+						text += child.value;
+						isJSX = isJSX || child.value.includes('{');
+					}
 				});
 
+
 				node.properties = node.properties || {};
 				if (typeof node.properties.id !== 'string') {
-					node.properties.id = slugger.slug(text);
+					if (isJSX) {
+						// HACK: for ids that have JSX content, use $$slug helper to generate slug at runtime
+						node.properties.id = `$$slug(\`${text.replace(/\{/g, '${')}\`)`;
+						(node as any).type = 'raw';
+						(node as any).value = `<${node.tagName} id={${node.properties.id}}>${raw}</${node.tagName}>`;
+					} else {
+						node.properties.id = slugger.slug(text);
+					}
 				}
 
 				headers.push({ depth, slug: node.properties.id, text });
diff --git a/packages/markdown/remark/src/rehype-expressions.ts b/packages/markdown/remark/src/rehype-expressions.ts
index 26d04623d..f06f242e2 100644
--- a/packages/markdown/remark/src/rehype-expressions.ts
+++ b/packages/markdown/remark/src/rehype-expressions.ts
@@ -3,8 +3,14 @@ import { map } from 'unist-util-map';
 export default function rehypeExpressions(): any {
 	return function (node: any): any {
 		return map(node, (child) => {
+			if (child.type === 'text') {
+				return { ...child, type: 'raw' };
+			}
 			if (child.type === 'mdxTextExpression') {
-				return { type: 'text', value: `{${(child as any).value}}` };
+				return { type: 'raw', value: `{${(child as any).value}}` };
+			}
+			if (child.type === 'mdxFlowExpression') {
+				return { type: 'raw', value: `{${(child as any).value}}` };
 			}
 			return child;
 		});
diff --git a/packages/markdown/remark/src/rehype-jsx.ts b/packages/markdown/remark/src/rehype-jsx.ts
index cccbd5548..62eb977c0 100644
--- a/packages/markdown/remark/src/rehype-jsx.ts
+++ b/packages/markdown/remark/src/rehype-jsx.ts
@@ -8,19 +8,41 @@ export default function rehypeJsx(): any {
 				return { ...child, tagName: `${child.tagName}` };
 			}
 			if (MDX_ELEMENTS.has(child.type)) {
-				return {
-					...child,
-					type: 'element',
-					tagName: `${child.name}`,
-					properties: child.attributes.reduce((acc: any[], entry: any) => {
+				const attrs = child.attributes.reduce((acc: any[], entry: any) => {
 						let attr = entry.value;
 						if (attr && typeof attr === 'object') {
 							attr = `{${attr.value}}`;
+						} else if (attr && entry.type === 'mdxJsxExpressionAttribute') {
+							attr = `{${attr}}`
 						} else if (attr === null) {
-							attr = `{true}`;
+							attr = "";
+						} else if (typeof attr === 'string') {
+							attr = `"${attr}"`;
+						}
+						if (!entry.name) {
+							return acc + ` ${attr}`;
 						}
-						return Object.assign(acc, { [entry.name]: attr });
-					}, {}),
+						return acc + ` ${entry.name}${attr ? '=' : ''}${attr}`;
+					}, '');
+
+				if (child.children.length === 0) {
+					return {
+						type: 'raw',
+						value: `<${child.name}${attrs} />`
+					};
+				}
+				child.children.splice(0, 0, {
+					type: 'raw',
+					value: `\n<${child.name}${attrs}>`
+				})
+				child.children.push({
+					type: 'raw',
+					value: `</${child.name}>\n`
+				})
+				return {
+					...child,
+					type: 'element',
+					tagName: `Fragment`,
 				};
 			}
 			return child;
diff --git a/packages/markdown/remark/src/remark-expressions.ts b/packages/markdown/remark/src/remark-expressions.ts
deleted file mode 100644
index 8e7af19f3..000000000
--- a/packages/markdown/remark/src/remark-expressions.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-// Vite bug: dynamically import() modules needed for CJS. Cache in memory to keep side effects
-let mdxExpressionFromMarkdown: any;
-let mdxExpressionToMarkdown: any;
-
-export function remarkExpressions(this: any, options: any) {
-	let settings = options || {};
-	let data = this.data();
-
-	add('fromMarkdownExtensions', mdxExpressionFromMarkdown);
-	add('toMarkdownExtensions', mdxExpressionToMarkdown);
-
-	function add(field: any, value: any) {
-		/* istanbul ignore if - other extensions. */
-		if (data[field]) data[field].push(value);
-		else data[field] = [value];
-	}
-}
-
-export async function loadRemarkExpressions() {
-	if (!mdxExpressionFromMarkdown || !mdxExpressionToMarkdown) {
-		const mdastUtilMdxExpression = await import('mdast-util-mdx-expression');
-		mdxExpressionFromMarkdown = mdastUtilMdxExpression.mdxExpressionFromMarkdown;
-		mdxExpressionToMarkdown = mdastUtilMdxExpression.mdxExpressionToMarkdown;
-	}
-}
diff --git a/packages/markdown/remark/src/remark-jsx.ts b/packages/markdown/remark/src/remark-jsx.ts
deleted file mode 100644
index 637bac9ee..000000000
--- a/packages/markdown/remark/src/remark-jsx.ts
+++ /dev/null
@@ -1,31 +0,0 @@
-// Vite bug: dynamically import() modules needed for CJS. Cache in memory to keep side effects
-let mdxJsx: any;
-let mdxJsxFromMarkdown: any;
-let mdxJsxToMarkdown: any;
-
-export function remarkJsx(this: any, options: any) {
-	let settings = options || {};
-	let data = this.data();
-
-	// TODO this seems to break adding slugs, no idea why add('micromarkExtensions', mdxJsx({}));
-	add('fromMarkdownExtensions', mdxJsxFromMarkdown);
-	add('toMarkdownExtensions', mdxJsxToMarkdown);
-
-	function add(field: any, value: any) {
-		/* istanbul ignore if - other extensions. */
-		if (data[field]) data[field].push(value);
-		else data[field] = [value];
-	}
-}
-
-export async function loadRemarkJsx() {
-	if (!mdxJsx) {
-		const micromarkMdxJsx = await import('micromark-extension-mdx-jsx');
-		mdxJsx = micromarkMdxJsx.mdxJsx;
-	}
-	if (!mdxJsxFromMarkdown || !mdxJsxToMarkdown) {
-		const mdastUtilMdxJsx = await import('mdast-util-mdx-jsx');
-		mdxJsxFromMarkdown = mdastUtilMdxJsx.mdxJsxFromMarkdown;
-		mdxJsxToMarkdown = mdastUtilMdxJsx.mdxJsxToMarkdown;
-	}
-}
diff --git a/packages/markdown/remark/src/remark-mark-and-unravel.ts b/packages/markdown/remark/src/remark-mark-and-unravel.ts
new file mode 100644
index 000000000..4490e4a93
--- /dev/null
+++ b/packages/markdown/remark/src/remark-mark-and-unravel.ts
@@ -0,0 +1,81 @@
+//  https://github.com/mdx-js/mdx/blob/main/packages/mdx/lib/plugin/remark-mark-and-unravel.js
+/**
+ * @typedef {import('mdast').Root} Root
+ * @typedef {import('mdast').Content} Content
+ * @typedef {Root|Content} Node
+ * @typedef {Extract<Node, import('unist').Parent>} Parent
+ *
+ * @typedef {import('remark-mdx')} DoNotTouchAsThisImportItIncludesMdxInTree
+ */
+
+import {visit} from 'unist-util-visit'
+
+/**
+ * A tiny plugin that unravels `<p><h1>x</h1></p>` but also
+ * `<p><Component /></p>` (so it has no knowledge of “HTML”).
+ * It also marks JSX as being explicitly JSX, so when a user passes a `h1`
+ * component, it is used for `# heading` but not for `<h1>heading</h1>`.
+ *
+ * @type {import('unified').Plugin<Array<void>, Root>}
+ */
+export default function remarkMarkAndUnravel() {
+  return (tree: any) => {
+    visit(tree, (node, index, parent_) => {
+      const parent = /** @type {Parent} */ (parent_)
+      let offset = -1
+      let all = true
+      /** @type {boolean|undefined} */
+      let oneOrMore
+
+      if (parent && typeof index === 'number' && node.type === 'paragraph') {
+        const children = node.children
+
+        while (++offset < children.length) {
+          const child = children[offset]
+
+          if (
+            child.type === 'mdxJsxTextElement' ||
+            child.type === 'mdxTextExpression'
+          ) {
+            oneOrMore = true
+          } else if (
+            child.type === 'text' &&
+            /^[\t\r\n ]+$/.test(String(child.value))
+          ) {
+            // Empty.
+          } else {
+            all = false
+            break
+          }
+        }
+
+        if (all && oneOrMore) {
+          offset = -1
+
+          while (++offset < children.length) {
+            const child = children[offset]
+
+            if (child.type === 'mdxJsxTextElement') {
+              child.type = 'mdxJsxFlowElement'
+            }
+
+            if (child.type === 'mdxTextExpression') {
+              child.type = 'mdxFlowExpression'
+            }
+          }
+
+          parent.children.splice(index, 1, ...children)
+          return index
+        }
+      }
+
+      if (
+        node.type === 'mdxJsxFlowElement' ||
+        node.type === 'mdxJsxTextElement'
+      ) {
+        const data = node.data || (node.data = {})
+        data._mdxExplicitJsx = true
+      }
+    })
+  }
+}
diff --git a/packages/markdown/remark/src/remark-mdxish.ts b/packages/markdown/remark/src/remark-mdxish.ts
new file mode 100644
index 000000000..b5d41d228
--- /dev/null
+++ b/packages/markdown/remark/src/remark-mdxish.ts
@@ -0,0 +1,15 @@
+import {mdxjs} from 'micromark-extension-mdxjs'
+import { mdxFromMarkdown, mdxToMarkdown } from './mdast-util-mdxish.js'
+
+export default function remarkMdxish(this: any, options = {}) {
+  const data = this.data()
+
+  add('micromarkExtensions', mdxjs(options))
+  add('fromMarkdownExtensions', mdxFromMarkdown())
+  add('toMarkdownExtensions', mdxToMarkdown())
+
+  function add(field: string, value: unknown) {
+    const list = data[field] ? data[field] : (data[field] = [])
+    list.push(value)
+  }
+}
diff --git a/packages/markdown/remark/src/remark-shiki.ts b/packages/markdown/remark/src/remark-shiki.ts
index e00156cb5..0b51f07ff 100644
--- a/packages/markdown/remark/src/remark-shiki.ts
+++ b/packages/markdown/remark/src/remark-shiki.ts
@@ -11,7 +11,7 @@ import type { ShikiConfig } from './types.js';
 const highlighterCacheAsync = new Map<string, Promise<shiki.Highlighter>>();
 
 const remarkShiki = async (
-	{ langs, theme, wrap }: ShikiConfig,
+	{ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig,
 	scopedClassName?: string | null
 ) => {
 	const cacheID: string = typeof theme === 'string' ? theme : theme.name;
diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts
index 3aef31710..af9778c9a 100644
--- a/packages/markdown/remark/src/types.ts
+++ b/packages/markdown/remark/src/types.ts
@@ -20,22 +20,24 @@ export type RehypePlugin<PluginParameters extends any[] = any[]> = unified.Plugi
 export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlugin, any])[];
 
 export interface ShikiConfig {
-	langs: ILanguageRegistration[];
-	theme: Theme | IThemeRegistration;
-	wrap: boolean | null;
+	langs?: ILanguageRegistration[];
+	theme?: Theme | IThemeRegistration;
+	wrap?: boolean | null;
 }
 
 export interface AstroMarkdownOptions {
-	mode: 'md' | 'mdx';
-	drafts: boolean;
-	syntaxHighlight: 'shiki' | 'prism' | false;
-	shikiConfig: ShikiConfig;
-	remarkPlugins: RemarkPlugins;
-	rehypePlugins: RehypePlugins;
+	mode?: 'md' | 'mdx';
+	drafts?: boolean;
+	syntaxHighlight?: 'shiki' | 'prism' | false;
+	shikiConfig?: ShikiConfig;
+	remarkPlugins?: RemarkPlugins;
+	rehypePlugins?: RehypePlugins;
 }
 
 export interface MarkdownRenderingOptions extends AstroMarkdownOptions {
 	/** @internal */
+	fileURL?: URL;
+	/** @internal */
 	$?: {
 		scopedClassName: string | null;
 	};