Fix markdown issues (#208)

* Init fix/markdown

* Astro Markdown (#207)

* Add Astro Markdown to VSCode Extension

* Add Astro Markdown to Astro

* refactor: update astro-markdown example

* feat: remove embedded components from `.md` files

* fix: resolve `.md.astro` files at runtime

* chore: update markdown tests

* feat: add <Markdown> component

* chore: bump examples

* chore: update example

* fix: improve Markdown child handling

* feat: harden markdown support, add code fence support, add automatic dedenting

* chore: add weird markdown edge cases

* chore: update remote-markdown examples

* chore: add comment to Markdown.astro

* feat: improve markdown support (codefences, nested inside HTML)

* refactor: extract import specifier types to set

* refactor: conditionally import markdown renderer

* refactor: revert special-cased "astro/components"

* refactor: revert special-cased "astro/components"

* refactor: use astro/components/Markdown.astro

* refactor: remove `.md.astro` support in favor of Markdown component

* refactor: use regular .astro files

* refactor: remove unused code

* refactor: move Markdown inside Layout

* wip: markdown scoped styles

* feat: improve scoped styles in Markdown

* feat: micromark => remark ecosystem

* fix: markdown build

* fix: markdown build

* chore: add todo

* fix: collect headers text

* docs: add Markdown doc

* chore: add changeset

* docs: improve Markdown highlighting

* refactor: prefer Set

* refactor: exclude large unified deps

* docs: update markdown docs

Co-authored-by: Jonathan Neal <jonathantneal@hotmail.com>

* chore: remove extra markdown deps

* perf: optimize markdown

* fix: unified/rehype deps

* temp: fix markdown test

* test: add TODO comment

* fix: do not namespace frontmatter, just astro metadata

* test: fix astro-markdown test

* test: add realworld markdown example

* fix: prism language bug

* docs: update markdown docs

* chore: bump dependencies

* fix: escape codespan

* fix: unterminated string literal

* fix(vscode): inline dependencies

* fix(vscode): dependencies

* feat(vscode): embedded markdown

* feat: add Markdown syntax highlighting

* chore: improve markdown example

* fix: markdown example

* feat: highlighting improvements

* chore: add changeset

* fix: CodeBlock => CodeSpan

* chore: get astro-markdown example running

Co-authored-by: Jonathan Neal <jonathantneal@hotmail.com>

5 files changed, 76 insertions, 3 deletions

diff --git a/packages/astro-parser/src/parse/state/codefence.ts b/packages/astro-parser/src/parse/state/codefence.ts
new file mode 100644
index 000000000..d5b498a0f
--- /dev/null
+++ b/packages/astro-parser/src/parse/state/codefence.ts
@@ -0,0 +1,38 @@
+// @ts-nocheck
+import { Parser } from '../index.js';
+
+export default function codefence(parser: Parser) {
+  const start = parser.index;
+  const open = parser.match_regex(/[`~]{3,}/);
+  parser.index += open!.length;
+
+  let raw = open + '';
+
+  while (parser.index < parser.template.length && !parser.match(open)) {
+    raw += parser.template[parser.index++];
+  }
+
+  parser.eat(open, true);
+  raw += open;
+  const trailingWhitespace = parser.read_until(/\S/);
+  const { metadata, data } = extractCodeFence(raw);
+
+  const node = {
+    start,
+    end: parser.index,
+    type: 'CodeFence',
+    raw: `${raw}` + trailingWhitespace,
+    metadata,
+    data
+  };
+
+  parser.current().children.push(node);
+}
+
+/** Extract attributes on first line */
+function extractCodeFence(str: string) {
+  const [_, leadingLine] = str.match(/(^[^\n]*\r?\n)/m) ?? ['', ''];
+  const metadata = leadingLine.trim();
+  const data = str.slice(leadingLine.length);
+  return { metadata, data };
+}
diff --git a/packages/astro-parser/src/parse/state/codespan.ts b/packages/astro-parser/src/parse/state/codespan.ts
new file mode 100644
index 000000000..b685800a7
--- /dev/null
+++ b/packages/astro-parser/src/parse/state/codespan.ts
@@ -0,0 +1,25 @@
+// @ts-nocheck
+import { Parser } from '../index.js';
+
+export default function codespan(parser: Parser) {
+  const start = parser.index;
+  const open = parser.match_regex(/(?<!\\)`{1,2}/);
+  parser.index += open!.length;
+
+  let raw = open;
+  while (parser.index < parser.template.length && !parser.match(open)) {
+    raw += parser.template[parser.index++];
+  }
+  parser.eat(open, true);
+  raw += open;
+
+  const node = {
+    start,
+    end: parser.index,
+    type: 'CodeSpan',
+    raw,
+    data: raw?.slice(open?.length, open?.length * -1).replace(/^ /, '').replace(/ $/, '')
+  };
+
+  parser.current().children.push(node);
+}
diff --git a/packages/astro-parser/src/parse/state/fragment.ts b/packages/astro-parser/src/parse/state/fragment.ts
index 97398b227..d3b30f329 100644
--- a/packages/astro-parser/src/parse/state/fragment.ts
+++ b/packages/astro-parser/src/parse/state/fragment.ts
@@ -2,6 +2,8 @@ import tag from './tag.js';
 import setup from './setup.js';
 import mustache from './mustache.js';
 import text from './text.js';
+import codefence from './codefence.js';
+import codespan from './codespan.js';
 import { Parser } from '../index.js';
 
 export default function fragment(parser: Parser) {
@@ -9,6 +11,15 @@ export default function fragment(parser: Parser) {
     return setup;
   }
 
+  // Fenced code blocks are pretty complex in the GFM spec
+  // https://github.github.com/gfm/#fenced-code-blocks
+  if (parser.match_regex(/[`~]{3,}/)) {
+    return codefence;
+  }
+  if (parser.match_regex(/(?<!\\)`{1,2}/)) {
+    return codespan;
+  }
+
   if (parser.match('<')) {
     return tag;
   }
diff --git a/packages/astro-parser/src/parse/state/tag.ts b/packages/astro-parser/src/parse/state/tag.ts
index 65bb93b4b..b8c3e63ad 100644
--- a/packages/astro-parser/src/parse/state/tag.ts
+++ b/packages/astro-parser/src/parse/state/tag.ts
@@ -1,7 +1,6 @@
 // @ts-nocheck
 
 import read_expression from '../read/expression.js';
-import read_script from '../read/script.js';
 import read_style from '../read/style.js';
 import { decode_character_references, closing_tag_omitted } from '../utils/html.js';
 import { is_void } from '../../utils/names.js';
@@ -518,7 +517,7 @@ function read_attribute_value(parser: Parser) {
   return value;
 }
 
-function read_sequence(parser: Parser, done: () => boolean): TemplateNode[] {
+export function read_sequence(parser: Parser, done: () => boolean): TemplateNode[] {
   let current_chunk: Text = {
     start: parser.index,
     end: null,
diff --git a/packages/astro-parser/src/parse/state/text.ts b/packages/astro-parser/src/parse/state/text.ts
index cca83f2d4..eac810a0a 100644
--- a/packages/astro-parser/src/parse/state/text.ts
+++ b/packages/astro-parser/src/parse/state/text.ts
@@ -8,7 +8,7 @@ export default function text(parser: Parser) {
 
   let data = '';
 
-  while (parser.index < parser.template.length && !parser.match('---') && !parser.match('<') && !parser.match('{')) {
+  while (parser.index < parser.template.length && !parser.match('---') && !parser.match('<') && !parser.match('{') && !parser.match('`')) {
     data += parser.template[parser.index++];
   }