Remark and rehype plugins (#562)

* remark plugins

* remove unused dependency

* enable codeblocks

* backward compatibility with remark-code-titles

* add support for rehype plugins

* add proper types for plugins

* fixes after review

- connect plugins by name
- make plugins configurable
- connect gfm and footnotes if no plugins provided from config
- add more plugins to example

* update and rename example

* add documentation for markdown plugins

* chore: rename with-markdown-plugins example

* chore: restructure dependencies

* feat: add back smartypants, fix mdx expressions

* chore: remove log

* test: add markdown plugin tests

* chore: add changeset

* docs: update markdown doc

Co-authored-by: Nate Moore <nate@skypack.dev>

7 files changed, 121 insertions, 30 deletions

diff --git a/packages/markdown-support/package.json b/packages/markdown-support/package.json
index 8be2aa4a4..550fef7a7 100644
--- a/packages/markdown-support/package.json
+++ b/packages/markdown-support/package.json
@@ -2,7 +2,7 @@
   "name": "@astrojs/markdown-support",
   "version": "0.1.2",
   "main": "./dist/index.js",
-  "type": "commonjs",
+  "type": "module",
   "repository": {
     "type": "git",
     "url": "https://github.com/snowpackjs/astro.git",
@@ -16,6 +16,18 @@
     "build": "astro-scripts build --format cjs \"src/**/*.ts\" && tsc -p tsconfig.json",
     "dev": "astro-scripts dev \"src/**/*.ts\""
   },
+  "dependencies": {
+    "@silvenon/remark-smartypants": "^1.0.0",
+    "gray-matter": "^4.0.2",
+    "mdast-util-mdx-expression": "^1.0.0",
+    "micromark-extension-mdx-expression": "^1.0.0",
+    "remark-footnotes": "^3.0.0",
+    "remark-gfm": "^1.0.0",
+    "remark-parse": "^9.0.0",
+    "remark-rehype": "^8.1.0",
+    "unified": "^9.2.1",
+    "unist-util-map": "^3.0.0"
+  },
   "devDependencies": {
     "@types/github-slugger": "^1.3.0",
     "github-slugger": "^1.3.0",
diff --git a/packages/markdown-support/src/index.ts b/packages/markdown-support/src/index.ts
index 08f171c3c..f311efa7c 100644
--- a/packages/markdown-support/src/index.ts
+++ b/packages/markdown-support/src/index.ts
@@ -1,62 +1,67 @@
-import type { AstroMarkdownOptions } from './types';
+import type { AstroMarkdownOptions, MarkdownRenderingOptions } from './types';
 
 import createCollectHeaders from './rehype-collect-headers.js';
 import scopedStyles from './remark-scoped-styles.js';
-import { remarkCodeBlock, rehypeCodeBlock } from './codeblock.js';
+import remarkExpressions from './remark-expressions.js';
+import rehypeExpressions from './rehype-expressions.js';
+import { rehypeCodeBlock } from './codeblock.js';
+import { loadPlugins } from './load-plugins.js';
 import raw from 'rehype-raw';
 
 import unified from 'unified';
 import markdown from 'remark-parse';
 import markdownToHtml from 'remark-rehype';
-// import smartypants from '@silvenon/remark-smartypants';
 import rehypeStringify from 'rehype-stringify';
 
-export interface MarkdownRenderingOptions extends Partial<AstroMarkdownOptions> {
-  $?: {
-    scopedClassName: string | null;
-  };
-  mode: 'md' | 'astro-md';
-}
+export { AstroMarkdownOptions, MarkdownRenderingOptions };
 
 /** Internal utility for rendering a full markdown file and extracting Frontmatter data */
 export async function renderMarkdownWithFrontmatter(contents: string, opts?: MarkdownRenderingOptions | null) {
   // Dynamic import to ensure that "gray-matter" isn't built by Snowpack
   const { default: matter } = await import('gray-matter');
   const { data: frontmatter, content } = matter(contents);
-  const value = await renderMarkdown(content, { ...opts, mode: 'md' });
+  const value = await renderMarkdown(content, opts);
   return { ...value, frontmatter };
 }
 
 /** Shared utility for rendering markdown */
 export async function renderMarkdown(content: string, opts?: MarkdownRenderingOptions | null) {
-  const { $: { scopedClassName = null } = {}, mode = 'astro-md', footnotes: useFootnotes = true, gfm: useGfm = true } = opts ?? {};
+  const { $: { scopedClassName = null } = {}, footnotes: useFootnotes = true, gfm: useGfm = true, remarkPlugins = [], rehypePlugins = [] } = opts ?? {};
   const { headers, rehypeCollectHeaders } = createCollectHeaders();
+  let parser = unified().use(markdown).use([remarkExpressions, { addResult: true }]);
 
-  let parser = unified().use(markdown).use(remarkCodeBlock());
+  if (remarkPlugins.length === 0) {
+    if (useGfm) {
+      remarkPlugins.push('remark-gfm');
+    }
 
-  if (scopedClassName) {
-    parser = parser.use(scopedStyles(scopedClassName));
-  }
+    if (useFootnotes) {
+      remarkPlugins.push('remark-footnotes');
+    }
 
-  if (useGfm) {
-    const { default: gfm } = await import('remark-gfm');
-    parser = parser.use(gfm);
+    remarkPlugins.push('@silvenon/remark-smartypants');
   }
+  const loadedRemarkPlugins = await Promise.all(loadPlugins(remarkPlugins));
+  const loadedRehypePlugins = await Promise.all(loadPlugins(rehypePlugins));
 
-  if (useFootnotes) {
-    const { default: footnotes } = await import('remark-footnotes');
-    parser = parser.use(footnotes);
+  loadedRemarkPlugins.forEach(([plugin, opts]) => {
+    parser.use(plugin, opts);
+  });
+
+  if (scopedClassName) {
+    parser.use(scopedStyles(scopedClassName));
   }
 
+  parser.use(markdownToHtml, { allowDangerousHtml: true, passThrough: ['raw', 'mdxTextExpression'] });
+  parser.use(rehypeExpressions);
+
+  loadedRehypePlugins.forEach(([plugin, opts]) => {
+    parser.use(plugin, opts);
+  });
+
   let result: string;
   try {
-    const vfile = await parser
-      .use(markdownToHtml, { allowDangerousHtml: true, passThrough: ['raw'] })
-      .use(raw)
-      .use(rehypeCollectHeaders)
-      .use(rehypeCodeBlock())
-      .use(rehypeStringify)
-      .process(content);
+    const vfile = await parser.use(raw).use(rehypeCollectHeaders).use(rehypeCodeBlock()).use(rehypeStringify, { entities: { useNamedReferences: true }}).process(content);
     result = vfile.contents.toString();
   } catch (err) {
     throw err;
diff --git a/packages/markdown-support/src/load-plugins.ts b/packages/markdown-support/src/load-plugins.ts
new file mode 100644
index 000000000..52bc287f8
--- /dev/null
+++ b/packages/markdown-support/src/load-plugins.ts
@@ -0,0 +1,27 @@
+import unified from "unified";
+import type { Plugin, UnifiedPluginImport } from "./types";
+
+async function importPlugin(p: string | UnifiedPluginImport): UnifiedPluginImport {
+  if (typeof p === 'string') {
+    return await import(p);
+  }
+
+  return await p;
+}
+
+export function loadPlugins(items: Plugin[]): Promise<[unified.Plugin] | [unified.Plugin, unified.Settings]>[] {
+  return items.map((p) => {
+    return new Promise((resolve, reject) => {
+      if (Array.isArray(p)) {
+        const [plugin, opts] = p;
+        return importPlugin(plugin)
+          .then((m) => resolve([m.default, opts]))
+          .catch((e) => reject(e));
+      }
+
+      return importPlugin(p)
+      .then((m) => resolve([m.default]))
+        .catch((e) => reject(e));
+    });
+  });
+}
diff --git a/packages/markdown-support/src/rehype-collect-headers.ts b/packages/markdown-support/src/rehype-collect-headers.ts
index edfcd29bc..de9b78692 100644
--- a/packages/markdown-support/src/rehype-collect-headers.ts
+++ b/packages/markdown-support/src/rehype-collect-headers.ts
@@ -14,11 +14,13 @@ export default function createCollectHeaders() {
     depth = Number.parseInt(depth);
 
     let text = '';
+
     visit(node, 'text', (child) => {
       text += child.value;
     });
 
-    let slug = slugger.slug(text);
+    let slug = node.properties.id || slugger.slug(text);
+
     node.properties = node.properties || {};
     node.properties.id = slug;
     headers.push({ depth, slug, text });
diff --git a/packages/markdown-support/src/rehype-expressions.ts b/packages/markdown-support/src/rehype-expressions.ts
new file mode 100644
index 000000000..2762f54fc
--- /dev/null
+++ b/packages/markdown-support/src/rehype-expressions.ts
@@ -0,0 +1,12 @@
+import { map } from 'unist-util-map'
+
+export default function rehypeExpressions(): any {
+  return function(node: any): any {
+    return map(node, (child) => {
+      if (child.type === 'mdxTextExpression') {
+        return { type: 'text', value: `{${child.value}}` }
+      }
+      return child;
+    })
+  }
+}
diff --git a/packages/markdown-support/src/remark-expressions.ts b/packages/markdown-support/src/remark-expressions.ts
new file mode 100644
index 000000000..1cdb37894
--- /dev/null
+++ b/packages/markdown-support/src/remark-expressions.ts
@@ -0,0 +1,19 @@
+import {mdxExpression} from 'micromark-extension-mdx-expression'
+import {mdxExpressionFromMarkdown, mdxExpressionToMarkdown} from 'mdast-util-mdx-expression'
+
+function remarkExpressions(this: any, options: any) {
+  let settings = options || {}
+  let data = this.data()
+
+  add('micromarkExtensions', mdxExpression({}))
+  add('fromMarkdownExtensions', mdxExpressionFromMarkdown)
+  add('toMarkdownExtensions', mdxExpressionToMarkdown)
+
+  function add(field: any, value: any) {
+    /* istanbul ignore if - other extensions. */
+    if (data[field]) data[field].push(value)
+    else data[field] = [value]
+  }
+}
+
+export default remarkExpressions;
diff --git a/packages/markdown-support/src/types.ts b/packages/markdown-support/src/types.ts
index b69f7fc28..6df601ae4 100644
--- a/packages/markdown-support/src/types.ts
+++ b/packages/markdown-support/src/types.ts
@@ -1,6 +1,20 @@
+import unified from 'unified';
+
+export type UnifiedPluginImport = Promise<{ default: unified.Plugin }>;
+export type Plugin = string | [string, unified.Settings] | UnifiedPluginImport | [UnifiedPluginImport, unified.Settings];
+
 export interface AstroMarkdownOptions {
   /** Enable or disable footnotes syntax extension */
   footnotes: boolean;
   /** Enable or disable GitHub-flavored Markdown syntax extension */
   gfm: boolean;
+  remarkPlugins: Plugin[];
+  rehypePlugins: Plugin[];
+}
+
+export interface MarkdownRenderingOptions extends Partial<AstroMarkdownOptions> {
+  /** @internal */
+  $?: {
+    scopedClassName: string | null;
+  };
 }