diff options
Diffstat (limited to 'packages/integrations/markdoc/src')
18 files changed, 1835 insertions, 0 deletions
diff --git a/packages/integrations/markdoc/src/config.ts b/packages/integrations/markdoc/src/config.ts new file mode 100644 index 000000000..c62bfebab --- /dev/null +++ b/packages/integrations/markdoc/src/config.ts @@ -0,0 +1,53 @@ +import { isRelativePath } from '@astrojs/internal-helpers/path'; +import type { + Config, + ConfigType as MarkdocConfig, + MaybePromise, + NodeType, + Schema, +} from '@markdoc/markdoc'; +import _Markdoc from '@markdoc/markdoc'; +import type { AstroInstance } from 'astro'; +import { heading } from './heading-ids.js'; +import { componentConfigSymbol } from './utils.js'; + +export type Render = ComponentConfig | AstroInstance['default'] | string; +export type ComponentConfig = { + type: 'package' | 'local'; + path: string; + namedExport?: string; + [componentConfigSymbol]: true; +}; + +export type AstroMarkdocConfig<C extends Record<string, any> = Record<string, any>> = Omit< + MarkdocConfig, + 'tags' | 'nodes' +> & + Partial<{ + tags: Record<string, Schema<Config, Render>>; + nodes: Partial<Record<NodeType, Schema<Config, Render>>>; + ctx: C; + extends: MaybePromise<ResolvedAstroMarkdocConfig>[]; + }>; + +export type ResolvedAstroMarkdocConfig = Omit<AstroMarkdocConfig, 'extends'>; + +export const Markdoc = _Markdoc; +export const nodes = { ...Markdoc.nodes, heading }; + +export function defineMarkdocConfig(config: AstroMarkdocConfig): AstroMarkdocConfig { + return config; +} + +export function component(pathnameOrPkgName: string, namedExport?: string): ComponentConfig { + return { + type: isNpmPackageName(pathnameOrPkgName) ? 'package' : 'local', + path: pathnameOrPkgName, + namedExport, + [componentConfigSymbol]: true, + }; +} + +function isNpmPackageName(pathname: string) { + return !isRelativePath(pathname) && !pathname.startsWith('/'); +} diff --git a/packages/integrations/markdoc/src/content-entry-type.ts b/packages/integrations/markdoc/src/content-entry-type.ts new file mode 100644 index 000000000..998d8fbb5 --- /dev/null +++ b/packages/integrations/markdoc/src/content-entry-type.ts @@ -0,0 +1,422 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseFrontmatter } from '@astrojs/markdown-remark'; +import type { Config as MarkdocConfig, Node } from '@markdoc/markdoc'; +import Markdoc from '@markdoc/markdoc'; +import type { AstroConfig, ContentEntryType } from 'astro'; +import { emitESMImage } from 'astro/assets/utils'; +import type { Rollup, ErrorPayload as ViteErrorPayload } from 'vite'; +import type { ComponentConfig } from './config.js'; +import { htmlTokenTransform } from './html/transform/html-token-transform.js'; +import type { MarkdocConfigResult } from './load-config.js'; +import type { MarkdocIntegrationOptions } from './options.js'; +import { setupConfig } from './runtime.js'; +import { getMarkdocTokenizer } from './tokenizer.js'; +import { MarkdocError, isComponentConfig, isValidUrl, prependForwardSlash } from './utils.js'; + +export async function getContentEntryType({ + markdocConfigResult, + astroConfig, + options, +}: { + astroConfig: AstroConfig; + markdocConfigResult?: MarkdocConfigResult; + options?: MarkdocIntegrationOptions; +}): Promise<ContentEntryType> { + return { + extensions: ['.mdoc'], + getEntryInfo({ fileUrl, contents }) { + const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); + return { + data: parsed.frontmatter, + body: parsed.content.trim(), + slug: parsed.frontmatter.slug, + rawData: parsed.rawFrontmatter, + }; + }, + handlePropagation: true, + async getRenderModule({ contents, fileUrl, viteId }) { + const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); + const tokenizer = getMarkdocTokenizer(options); + let tokens = tokenizer.tokenize(parsed.content); + + if (options?.allowHTML) { + tokens = htmlTokenTransform(tokenizer, tokens); + } + + const ast = Markdoc.parse(tokens); + const userMarkdocConfig = markdocConfigResult?.config ?? {}; + const markdocConfigUrl = markdocConfigResult?.fileUrl; + const pluginContext = this; + const markdocConfig = await setupConfig(userMarkdocConfig, options); + const filePath = fileURLToPath(fileUrl); + raiseValidationErrors({ + ast, + /* Raised generics issue with Markdoc core https://github.com/markdoc/markdoc/discussions/400 */ + markdocConfig: markdocConfig as MarkdocConfig, + viteId, + astroConfig, + filePath, + }); + await resolvePartials({ + ast, + markdocConfig: markdocConfig as MarkdocConfig, + fileUrl, + allowHTML: options?.allowHTML, + tokenizer, + pluginContext, + root: astroConfig.root, + raisePartialValidationErrors: (partialAst, partialPath) => { + raiseValidationErrors({ + ast: partialAst, + markdocConfig: markdocConfig as MarkdocConfig, + viteId, + astroConfig, + filePath: partialPath, + }); + }, + }); + + const usedTags = getUsedTags(ast); + + let componentConfigByTagMap: Record<string, ComponentConfig> = {}; + // Only include component imports for tags used in the document. + // Avoids style and script bleed. + for (const tag of usedTags) { + const render = markdocConfig.tags?.[tag]?.render; + if (isComponentConfig(render)) { + componentConfigByTagMap[tag] = render; + } + } + let componentConfigByNodeMap: Record<string, ComponentConfig> = {}; + for (const [nodeType, schema] of Object.entries(markdocConfig.nodes ?? {})) { + const render = schema?.render; + if (isComponentConfig(render)) { + componentConfigByNodeMap[nodeType] = render; + } + } + + await emitOptimizedImages(ast.children, { + astroConfig, + pluginContext, + filePath, + }); + + const res = `import { Renderer } from '@astrojs/markdoc/components'; +import { createGetHeadings, createContentComponent } from '@astrojs/markdoc/runtime'; +${ + markdocConfigUrl + ? `import markdocConfig from ${JSON.stringify(fileURLToPath(markdocConfigUrl))};` + : 'const markdocConfig = {};' +} + +import { assetsConfig } from '@astrojs/markdoc/runtime-assets-config'; +markdocConfig.nodes = { ...assetsConfig.nodes, ...markdocConfig.nodes }; + +${getStringifiedImports(componentConfigByTagMap, 'Tag', astroConfig.root)} +${getStringifiedImports(componentConfigByNodeMap, 'Node', astroConfig.root)} + +const tagComponentMap = ${getStringifiedMap(componentConfigByTagMap, 'Tag')}; +const nodeComponentMap = ${getStringifiedMap(componentConfigByNodeMap, 'Node')}; + +const options = ${JSON.stringify(options)}; + +const stringifiedAst = ${JSON.stringify( + /* Double stringify to encode *as* stringified JSON */ JSON.stringify(ast), + )}; + +export const getHeadings = createGetHeadings(stringifiedAst, markdocConfig, options); +export const Content = createContentComponent( + Renderer, + stringifiedAst, + markdocConfig, + options, + tagComponentMap, + nodeComponentMap, +)`; + return { code: res }; + }, + contentModuleTypes: await fs.promises.readFile( + new URL('../template/content-module-types.d.ts', import.meta.url), + 'utf-8', + ), + }; +} + +/** + * Recursively resolve partial tags to their content. + * Note: Mutates the `ast` object directly. + */ +async function resolvePartials({ + ast, + fileUrl, + root, + tokenizer, + allowHTML, + markdocConfig, + pluginContext, + raisePartialValidationErrors, +}: { + ast: Node; + fileUrl: URL; + root: URL; + tokenizer: any; + allowHTML?: boolean; + markdocConfig: MarkdocConfig; + pluginContext: Rollup.PluginContext; + raisePartialValidationErrors: (ast: Node, filePath: string) => void; +}) { + const relativePartialPath = path.relative(fileURLToPath(root), fileURLToPath(fileUrl)); + for (const node of ast.walk()) { + if (node.type === 'tag' && node.tag === 'partial') { + const { file } = node.attributes; + if (!file) { + throw new MarkdocError({ + // Should be caught by Markdoc validation step. + message: `(Uncaught error) Partial tag requires a 'file' attribute`, + }); + } + + if (markdocConfig.partials?.[file]) continue; + + let partialPath: string; + let partialContents: string; + try { + const resolved = await pluginContext.resolve(file, fileURLToPath(fileUrl)); + let partialId = resolved?.id; + if (!partialId) { + const attemptResolveAsRelative = await pluginContext.resolve( + './' + file, + fileURLToPath(fileUrl), + ); + if (!attemptResolveAsRelative?.id) throw new Error(); + partialId = attemptResolveAsRelative.id; + } + + partialPath = fileURLToPath(new URL(prependForwardSlash(partialId), 'file://')); + partialContents = await fs.promises.readFile(partialPath, 'utf-8'); + } catch { + throw new MarkdocError({ + message: [ + `**${String(relativePartialPath)}** contains invalid content:`, + `Could not read partial file \`${file}\`. Does the file exist?`, + ].join('\n'), + }); + } + if (pluginContext.meta.watchMode) pluginContext.addWatchFile(partialPath); + let partialTokens = tokenizer.tokenize(partialContents); + if (allowHTML) { + partialTokens = htmlTokenTransform(tokenizer, partialTokens); + } + const partialAst = Markdoc.parse(partialTokens); + raisePartialValidationErrors(partialAst, partialPath); + await resolvePartials({ + ast: partialAst, + root, + fileUrl: pathToFileURL(partialPath), + tokenizer, + allowHTML, + markdocConfig, + pluginContext, + raisePartialValidationErrors, + }); + + Object.assign(node, partialAst); + } + } +} + +function raiseValidationErrors({ + ast, + markdocConfig, + viteId, + astroConfig, + filePath, +}: { + ast: Node; + markdocConfig: MarkdocConfig; + viteId: string; + astroConfig: AstroConfig; + filePath: string; +}) { + const validationErrors = Markdoc.validate(ast, markdocConfig).filter((e) => { + return ( + (e.error.level === 'error' || e.error.level === 'critical') && + // Ignore `variable-undefined` errors. + // Variables can be configured at runtime, + // so we cannot validate them at build time. + e.error.id !== 'variable-undefined' && + // Ignore missing partial errors. + // We will resolve these in `resolvePartials`. + !(e.error.id === 'attribute-value-invalid' && /^Partial .+ not found/.test(e.error.message)) + ); + }); + + if (validationErrors.length) { + const rootRelativePath = path.relative(fileURLToPath(astroConfig.root), filePath); + throw new MarkdocError({ + message: [ + `**${String(rootRelativePath)}** contains invalid content:`, + ...validationErrors.map((e) => `- ${e.error.message}`), + ].join('\n'), + location: { + // Error overlay does not support multi-line or ranges. + // Just point to the first line. + line: validationErrors[0].lines[0], + file: viteId, + }, + }); + } +} + +function getUsedTags(markdocAst: Node) { + const tags = new Set<string>(); + const validationErrors = Markdoc.validate(markdocAst); + // Hack: run the validator with an empty config and look for 'tag-undefined'. + // This is our signal that a tag is being used! + for (const { error } of validationErrors) { + if (error.id === 'tag-undefined') { + const [, tagName] = /Undefined tag: '(.*)'/.exec(error.message) ?? []; + tags.add(tagName); + } + } + return tags; +} + +/** + * Emits optimized images, and appends the generated `src` to each AST node + * via the `__optimizedSrc` attribute. + */ +async function emitOptimizedImages( + nodeChildren: Node[], + ctx: { + pluginContext: Rollup.PluginContext; + filePath: string; + astroConfig: AstroConfig; + }, +) { + for (const node of nodeChildren) { + let isComponent = node.type === 'tag' && node.tag === 'image'; + // Support either a ![]() or {% image %} syntax, and handle the `src` attribute accordingly. + if ((node.type === 'image' || isComponent) && typeof node.attributes.src === 'string') { + let attributeName = isComponent ? 'src' : '__optimizedSrc'; + + // If the image isn't an URL or a link to public, try to resolve it. + if (shouldOptimizeImage(node.attributes.src)) { + // Attempt to resolve source with Vite. + // This handles relative paths and configured aliases + const resolved = await ctx.pluginContext.resolve(node.attributes.src, ctx.filePath); + + if (resolved?.id && fs.existsSync(new URL(prependForwardSlash(resolved.id), 'file://'))) { + const src = await emitESMImage( + resolved.id, + ctx.pluginContext.meta.watchMode, + !!ctx.astroConfig.experimental.svg, + ctx.pluginContext.emitFile, + ); + + const fsPath = resolved.id; + + if (src) { + // We cannot track images in Markdoc, Markdoc rendering always strips out the proxy. As such, we'll always + // assume that the image is referenced elsewhere, to be on safer side. + if (ctx.astroConfig.output === 'static') { + if (globalThis.astroAsset.referencedImages) + globalThis.astroAsset.referencedImages.add(fsPath); + } + + node.attributes[attributeName] = { ...src, fsPath }; + } + } else { + throw new MarkdocError({ + message: `Could not resolve image ${JSON.stringify( + node.attributes.src, + )} from ${JSON.stringify(ctx.filePath)}. Does the file exist?`, + }); + } + } else if (isComponent) { + // If the user is using the {% image %} tag, always pass the `src` attribute as `__optimizedSrc`, even if it's an external URL or absolute path. + // That way, the component can decide whether to optimize it or not. + node.attributes[attributeName] = node.attributes.src; + } + } + await emitOptimizedImages(node.children, ctx); + } +} + +function shouldOptimizeImage(src: string) { + // Optimize anything that is NOT external or an absolute path to `public/` + return !isValidUrl(src) && !src.startsWith('/'); +} + +/** + * Get stringified import statements for configured tags or nodes. + * `componentNamePrefix` is appended to the import name for namespacing. + * + * Example output: `import Tagaside from '/Users/.../src/components/Aside.astro';` + */ +function getStringifiedImports( + componentConfigMap: Record<string, ComponentConfig>, + componentNamePrefix: string, + root: URL, +) { + let stringifiedComponentImports = ''; + for (const [key, config] of Object.entries(componentConfigMap)) { + const importName = config.namedExport + ? `{ ${config.namedExport} as ${componentNamePrefix + toImportName(key)} }` + : componentNamePrefix + toImportName(key); + const resolvedPath = + config.type === 'local' ? fileURLToPath(new URL(config.path, root)) : config.path; + + stringifiedComponentImports += `import ${importName} from ${JSON.stringify(resolvedPath)};\n`; + } + return stringifiedComponentImports; +} + +function toImportName(unsafeName: string) { + // TODO: more checks that name is a safe JS variable name + return unsafeName.replace('-', '_'); +} + +/** + * Get a stringified map from tag / node name to component import name. + * This uses the same `componentNamePrefix` used by `getStringifiedImports()`. + * + * Example output: `{ aside: Tagaside, heading: Tagheading }` + */ +function getStringifiedMap( + componentConfigMap: Record<string, ComponentConfig>, + componentNamePrefix: string, +) { + let stringifiedComponentMap = '{'; + for (const key in componentConfigMap) { + stringifiedComponentMap += `${JSON.stringify(key)}: ${ + componentNamePrefix + toImportName(key) + },\n`; + } + stringifiedComponentMap += '}'; + return stringifiedComponentMap; +} + +/** + * Match YAML exception handling from Astro core errors + * @see 'astro/src/core/errors.ts' + */ +function safeParseFrontmatter(fileContents: string, filePath: string) { + try { + // empty with lines to preserve sourcemap location, but not `empty-with-spaces` + // because markdoc struggles with spaces + return parseFrontmatter(fileContents, { frontmatter: 'empty-with-lines' }); + } catch (e: any) { + if (e.name === 'YAMLException') { + const err: Error & ViteErrorPayload['err'] = e; + err.id = filePath; + err.loc = { file: e.id, line: e.mark.line + 1, column: e.mark.column }; + err.message = e.reason; + throw err; + } else { + throw e; + } + } +} diff --git a/packages/integrations/markdoc/src/extensions/prism.ts b/packages/integrations/markdoc/src/extensions/prism.ts new file mode 100644 index 000000000..721564871 --- /dev/null +++ b/packages/integrations/markdoc/src/extensions/prism.ts @@ -0,0 +1,21 @@ +import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter'; +import { unescapeHTML } from 'astro/runtime/server/index.js'; +import { type AstroMarkdocConfig, Markdoc } from '../config.js'; + +export default function prism(): AstroMarkdocConfig { + return { + nodes: { + fence: { + attributes: Markdoc.nodes.fence.attributes!, + transform({ attributes: { language, content } }) { + const { html, classLanguage } = runHighlighterWithAstro(language, content); + + // Use `unescapeHTML` to return `HTMLString` for Astro renderer to inline as HTML + return unescapeHTML( + `<pre class="${classLanguage}"><code class="${classLanguage}">${html}</code></pre>`, + ) as any; + }, + }, + }, + }; +} diff --git a/packages/integrations/markdoc/src/extensions/shiki.ts b/packages/integrations/markdoc/src/extensions/shiki.ts new file mode 100644 index 000000000..1102242fd --- /dev/null +++ b/packages/integrations/markdoc/src/extensions/shiki.ts @@ -0,0 +1,35 @@ +import { createShikiHighlighter } from '@astrojs/markdown-remark'; +import Markdoc from '@markdoc/markdoc'; +import type { ShikiConfig } from 'astro'; +import { unescapeHTML } from 'astro/runtime/server/index.js'; +import type { AstroMarkdocConfig } from '../config.js'; + +export default async function shiki(config?: ShikiConfig): Promise<AstroMarkdocConfig> { + const highlighter = await createShikiHighlighter({ + langs: config?.langs, + theme: config?.theme, + themes: config?.themes, + }); + + return { + nodes: { + fence: { + attributes: Markdoc.nodes.fence.attributes!, + async transform({ attributes }) { + // NOTE: The `meta` from fence code, e.g. ```js {1,3-4}, isn't quite supported by Markdoc. + // Only the `js` part is parsed as `attributes.language` and the rest is ignored. This means + // some Shiki transformers may not work correctly as it relies on the `meta`. + const lang = typeof attributes.language === 'string' ? attributes.language : 'plaintext'; + const html = await highlighter.codeToHtml(attributes.content, lang, { + wrap: config?.wrap, + defaultColor: config?.defaultColor, + transformers: config?.transformers, + }); + + // Use `unescapeHTML` to return `HTMLString` for Astro renderer to inline as HTML + return unescapeHTML(html) as any; + }, + }, + }, + }; +} diff --git a/packages/integrations/markdoc/src/heading-ids.ts b/packages/integrations/markdoc/src/heading-ids.ts new file mode 100644 index 000000000..9290a3db6 --- /dev/null +++ b/packages/integrations/markdoc/src/heading-ids.ts @@ -0,0 +1,77 @@ +import Markdoc, { + type Config as MarkdocConfig, + type RenderableTreeNode, + type Schema, +} from '@markdoc/markdoc'; +import Slugger from 'github-slugger'; +import { getTextContent } from './runtime.js'; +import { MarkdocError } from './utils.js'; + +function getSlug( + attributes: Record<string, any>, + children: RenderableTreeNode[], + headingSlugger: Slugger, +): string { + if (attributes.id && typeof attributes.id === 'string') { + return attributes.id; + } + const textContent = attributes.content ?? getTextContent(children); + let slug = headingSlugger.slug(textContent); + + if (slug.endsWith('-')) slug = slug.slice(0, -1); + return slug; +} + +type HeadingIdConfig = MarkdocConfig & { + ctx: { headingSlugger: Slugger }; +}; + +/* + Expose standalone node for users to import in their config. + Allows users to apply a custom `render: AstroComponent` + and spread our default heading attributes. +*/ +export const heading: Schema = { + children: ['inline'], + attributes: { + id: { type: String }, + level: { type: Number, required: true, default: 1 }, + }, + transform(node, config: HeadingIdConfig) { + const { level, ...attributes } = node.transformAttributes(config); + const children = node.transformChildren(config); + + if (!config.ctx?.headingSlugger) { + throw new MarkdocError({ + message: + 'Unexpected problem adding heading IDs to Markdoc file. Did you modify the `ctx.headingSlugger` property in your Markdoc config?', + }); + } + const slug = getSlug(attributes, children, config.ctx.headingSlugger); + + const render = config.nodes?.heading?.render ?? `h${level}`; + + const tagProps = + // For components, pass down `level` as a prop, + // alongside `__collectHeading` for our `headings` collector. + // Avoid accidentally rendering `level` as an HTML attribute otherwise! + typeof render === 'string' + ? { ...attributes, id: slug } + : { ...attributes, id: slug, __collectHeading: true, level }; + + return new Markdoc.Tag(render, tagProps, children); + }, +}; + +// Called internally to ensure `ctx` is generated per-file, instead of per-build. +export function setupHeadingConfig(): HeadingIdConfig { + const headingSlugger = new Slugger(); + return { + ctx: { + headingSlugger, + }, + nodes: { + heading, + }, + }; +} diff --git a/packages/integrations/markdoc/src/html/css/parse-inline-css-to-react.ts b/packages/integrations/markdoc/src/html/css/parse-inline-css-to-react.ts new file mode 100644 index 000000000..c35376641 --- /dev/null +++ b/packages/integrations/markdoc/src/html/css/parse-inline-css-to-react.ts @@ -0,0 +1,24 @@ +import { styleToObject } from './style-to-object.js'; + +export function parseInlineCSSToReactLikeObject( + css: string | undefined | null, +): React.CSSProperties | undefined { + if (typeof css === 'string') { + const cssObject: Record<string, string> = {}; + styleToObject(css, (originalCssDirective: string, value: string) => { + const reactCssDirective = convertCssDirectiveNameToReactCamelCase(originalCssDirective); + cssObject[reactCssDirective] = value; + }); + return cssObject; + } + + return undefined; +} + +function convertCssDirectiveNameToReactCamelCase(original: string): string { + // capture group 1 is the character to capitalize, the hyphen is omitted by virtue of being outside the capture group + const replaced = original.replace(/-([a-z\d])/gi, (_match, char) => { + return char.toUpperCase(); + }); + return replaced; +} diff --git a/packages/integrations/markdoc/src/html/css/parse-inline-styles.ts b/packages/integrations/markdoc/src/html/css/parse-inline-styles.ts new file mode 100644 index 000000000..fa3217c89 --- /dev/null +++ b/packages/integrations/markdoc/src/html/css/parse-inline-styles.ts @@ -0,0 +1,276 @@ +// @ts-nocheck +// https://github.com/remarkablemark/inline-style-parser + +/** + * @license MIT + * + * (The MIT License) + * + * Copyright (c) 2012 TJ Holowaychuk <tj@vision-media.ca> + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +// http://www.w3.org/TR/CSS21/grammar.html +// https://github.com/visionmedia/css-parse/pull/49#issuecomment-30088027 +const COMMENT_REGEX = /\/\*[^*]*\*+([^/*][^*]*\*+)*\//g; + +const NEWLINE_REGEX = /\n/g; +const WHITESPACE_REGEX = /^\s*/; + +// declaration +const PROPERTY_REGEX = /^([-#/*\\\w]+(\[[\da-z_-]+\])?)\s*/; +const COLON_REGEX = /^:\s*/; +// Disable eslint as we're not sure how to improve this regex yet +// eslint-disable-next-line regexp/no-super-linear-backtracking +const VALUE_REGEX = /^((?:'(?:\\'|.)*?'|"(?:\\"|.)*?"|\([^)]*\)|[^};])+)/; +const SEMICOLON_REGEX = /^[;\s]*/; + +// https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String/Trim#Polyfill +const TRIM_REGEX = /^\s+|\s+$/g; + +// strings +const NEWLINE = '\n'; +const FORWARD_SLASH = '/'; +const ASTERISK = '*'; +const EMPTY_STRING = ''; + +// types +const TYPE_COMMENT = 'comment'; +const TYPE_DECLARATION = 'declaration'; + +/** + * @param {String} style + * @param {Object} [options] + * @return {Object[]} + * @throws {TypeError} + * @throws {Error} + */ +export function parseInlineStyles(style, options) { + if (typeof style !== 'string') { + throw new TypeError('First argument must be a string'); + } + + if (!style) return []; + + options = options || {}; + + /** + * Positional. + */ + let lineno = 1; + let column = 1; + + /** + * Update lineno and column based on `str`. + * + * @param {String} str + */ + function updatePosition(str) { + let lines = str.match(NEWLINE_REGEX); + if (lines) lineno += lines.length; + let i = str.lastIndexOf(NEWLINE); + column = ~i ? str.length - i : column + str.length; + } + + /** + * Mark position and patch `node.position`. + * + * @return {Function} + */ + function position() { + let start = { line: lineno, column: column }; + return function (node) { + node.position = new Position(start); + whitespace(); + return node; + }; + } + + /** + * Store position information for a node. + * + * @constructor + * @property {Object} start + * @property {Object} end + * @property {undefined|String} source + */ + function Position(start) { + this.start = start; + this.end = { line: lineno, column: column }; + this.source = options.source; + } + + /** + * Non-enumerable source string. + */ + Position.prototype.content = style; + + const errorsList = []; + + /** + * Error `msg`. + * + * @param {String} msg + * @throws {Error} + */ + function error(msg) { + const err = new Error(options.source + ':' + lineno + ':' + column + ': ' + msg); + err.reason = msg; + err.filename = options.source; + err.line = lineno; + err.column = column; + err.source = style; + + if (options.silent) { + errorsList.push(err); + } else { + throw err; + } + } + + /** + * Match `re` and return captures. + * + * @param {RegExp} re + * @return {undefined|Array} + */ + function match(re) { + const m = re.exec(style); + if (!m) return; + const str = m[0]; + updatePosition(str); + style = style.slice(str.length); + return m; + } + + /** + * Parse whitespace. + */ + function whitespace() { + match(WHITESPACE_REGEX); + } + + /** + * Parse comments. + * + * @param {Object[]} [rules] + * @return {Object[]} + */ + function comments(rules) { + let c; + rules = rules || []; + while ((c = comment())) { + if (c !== false) { + rules.push(c); + } + } + return rules; + } + + /** + * Parse comment. + * + * @return {Object} + * @throws {Error} + */ + function comment() { + const pos = position(); + if (FORWARD_SLASH != style.charAt(0) || ASTERISK != style.charAt(1)) return; + + let i = 2; + while ( + EMPTY_STRING != style.charAt(i) && + (ASTERISK != style.charAt(i) || FORWARD_SLASH != style.charAt(i + 1)) + ) { + ++i; + } + i += 2; + + if (EMPTY_STRING === style.charAt(i - 1)) { + return error('End of comment missing'); + } + + const str = style.slice(2, i - 2); + column += 2; + updatePosition(str); + style = style.slice(i); + column += 2; + + return pos({ + type: TYPE_COMMENT, + comment: str, + }); + } + + /** + * Parse declaration. + * + * @return {Object} + * @throws {Error} + */ + function declaration() { + const pos = position(); + + // prop + const prop = match(PROPERTY_REGEX); + if (!prop) return; + comment(); + + // : + if (!match(COLON_REGEX)) return error("property missing ':'"); + + // val + const val = match(VALUE_REGEX); + + const ret = pos({ + type: TYPE_DECLARATION, + property: trim(prop[0].replace(COMMENT_REGEX, EMPTY_STRING)), + value: val ? trim(val[0].replace(COMMENT_REGEX, EMPTY_STRING)) : EMPTY_STRING, + }); + + // ; + match(SEMICOLON_REGEX); + + return ret; + } + + /** + * Parse declarations. + * + * @return {Object[]} + */ + function declarations() { + const decls = []; + + comments(decls); + + // declarations + let decl; + while ((decl = declaration())) { + if (decl !== false) { + decls.push(decl); + comments(decls); + } + } + + return decls; + } + + whitespace(); + return declarations(); +} + +/** + * Trim `str`. + * + * @param {String} str + * @return {String} + */ +function trim(str) { + return str ? str.replace(TRIM_REGEX, EMPTY_STRING) : EMPTY_STRING; +} diff --git a/packages/integrations/markdoc/src/html/css/style-to-object.ts b/packages/integrations/markdoc/src/html/css/style-to-object.ts new file mode 100644 index 000000000..084c93c93 --- /dev/null +++ b/packages/integrations/markdoc/src/html/css/style-to-object.ts @@ -0,0 +1,70 @@ +// @ts-nocheck +// https://github.com/remarkablemark/style-to-object + +/** + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Menglin "Mark" Xu <mark@remarkablemark.org> + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +import { parseInlineStyles } from './parse-inline-styles.js'; + +/** + * Parses inline style to object. + * + * @example + * // returns { 'line-height': '42' } + * styleToObject('line-height: 42;'); + * + * @param {String} style - The inline style. + * @param {Function} [iterator] - The iterator function. + * @return {null|Object} + */ +export function styleToObject(style, iterator) { + let output = null; + if (!style || typeof style !== 'string') { + return output; + } + + let declaration; + let declarations = parseInlineStyles(style); + let hasIterator = typeof iterator === 'function'; + let property; + let value; + + for (let i = 0, len = declarations.length; i < len; i++) { + declaration = declarations[i]; + property = declaration.property; + value = declaration.value; + + if (hasIterator) { + iterator(property, value, declaration); + } else if (value) { + output || (output = {}); + output[property] = value; + } + } + + return output; +} diff --git a/packages/integrations/markdoc/src/html/index.ts b/packages/integrations/markdoc/src/html/index.ts new file mode 100644 index 000000000..3f947736c --- /dev/null +++ b/packages/integrations/markdoc/src/html/index.ts @@ -0,0 +1,2 @@ +export { htmlTag } from './tagdefs/html.tag.js'; +export { htmlTokenTransform } from './transform/html-token-transform.js'; diff --git a/packages/integrations/markdoc/src/html/tagdefs/html.tag.ts b/packages/integrations/markdoc/src/html/tagdefs/html.tag.ts new file mode 100644 index 000000000..0c094227d --- /dev/null +++ b/packages/integrations/markdoc/src/html/tagdefs/html.tag.ts @@ -0,0 +1,69 @@ +import type { Config, Schema } from '@markdoc/markdoc'; +import Markdoc from '@markdoc/markdoc'; + +const booleanAttributes = new Set([ + 'allowfullscreen', + 'async', + 'autofocus', + 'autoplay', + 'checked', + 'controls', + 'default', + 'defer', + 'disabled', + 'disablepictureinpicture', + 'disableremoteplayback', + 'download', + 'formnovalidate', + 'hidden', + 'inert', + 'ismap', + 'itemscope', + 'loop', + 'multiple', + 'muted', + 'nomodule', + 'novalidate', + 'open', + 'playsinline', + 'readonly', + 'required', + 'reversed', + 'selected', +]); + +// local +import { parseInlineCSSToReactLikeObject } from '../css/parse-inline-css-to-react.js'; + +// a Markdoc tag that will render a given HTML element and its attributes, as produced by the htmlTokenTransform function +export const htmlTag: Schema<Config, never> = { + attributes: { + name: { type: String, required: true }, + attrs: { type: Object }, + }, + + transform(node, config) { + const { name, attrs: unsafeAttributes } = node.attributes; + const children = node.transformChildren(config); + + // pull out any "unsafe" attributes which need additional processing + const { style, ...safeAttributes } = unsafeAttributes as Record<string, unknown>; + + // Convert boolean attributes to boolean literals + for (const [key, value] of Object.entries(safeAttributes)) { + if (booleanAttributes.has(key)) { + // If the attribute exists, ensure its value is a boolean + safeAttributes[key] = value === '' || value === true || value === 'true'; + } + } + + // if the inline "style" attribute is present we need to parse the HTML into a react-like React.CSSProperties object + if (typeof style === 'string') { + const styleObject = parseInlineCSSToReactLikeObject(style); + safeAttributes.style = styleObject; + } + + // create a Markdoc Tag for the given HTML node with the HTML attributes and children + return new Markdoc.Tag(name, safeAttributes, children); + }, +}; diff --git a/packages/integrations/markdoc/src/html/transform/html-token-transform.ts b/packages/integrations/markdoc/src/html/transform/html-token-transform.ts new file mode 100644 index 000000000..b80595f97 --- /dev/null +++ b/packages/integrations/markdoc/src/html/transform/html-token-transform.ts @@ -0,0 +1,249 @@ +import type { Tokenizer } from '@markdoc/markdoc'; +import { Parser } from 'htmlparser2'; +// @ts-expect-error This type isn't exported +// biome-ignore lint/correctness/noUnusedImports: not correctly detected because type isn't exported +import type * as Token from 'markdown-it/lib/token'; + +export function htmlTokenTransform(tokenizer: Tokenizer, tokens: Token[]): Token[] { + const output: Token[] = []; + + // hold a lazy buffer of text and process it only when necessary + let textBuffer = ''; + + let inCDATA = false; + + const appendText = (text: string) => { + textBuffer += text; + }; + + // process the current text buffer w/ Markdoc's Tokenizer for tokens + const processTextBuffer = () => { + if (textBuffer.length > 0) { + // tokenize the text buffer to look for structural markup tokens + const toks = tokenizer.tokenize(textBuffer); + + // when we tokenize some raw text content, it's basically treated like Markdown, and will result in a paragraph wrapper, which we don't want + // in this scenario, we just want to generate a text token, but, we have to tokenize it in case there's other structural markup + if (toks.length === 3) { + const first = toks[0]; + const second = toks[1]; + const third: Token | undefined = toks.at(2); + + if ( + first.type === 'paragraph_open' && + second.type === 'inline' && + third && + third.type === 'paragraph_close' && + Array.isArray(second.children) + ) { + for (const tok of second.children as Token[]) { + // if the given token is a 'text' token and its trimmed content is the same as the pre-tokenized text buffer, use the original + // text buffer instead to preserve leading/trailing whitespace that is lost during tokenization of pure text content + if (tok.type === 'text') { + if (tok.content.trim() == textBuffer.trim()) { + tok.content = textBuffer; + } + } + output.push(tok); + } + } else { + // some other markup that happened to be 3 tokens, push tokens as-is + for (const tok of toks) { + output.push(tok); + } + } + } else { + // some other tokenized markup, push tokens as-is + for (const tok of toks) { + output.push(tok); + } + } + + // reset the current lazy text buffer + textBuffer = ''; + } + }; + + // create an incremental HTML parser that tracks HTML tag open, close and text content + const parser = new Parser( + { + oncdatastart() { + inCDATA = true; + }, + + oncdataend() { + inCDATA = false; + }, + + // when an HTML tag opens... + onopentag(name, attrs) { + // process any buffered text to be treated as text node before the currently opening HTML tag + processTextBuffer(); + + // push an 'html-tag' 'tag_open' Markdoc node instance for the currently opening HTML tag onto the resulting Token stack + output.push({ + type: 'tag_open', + nesting: 1, + meta: { + tag: 'html-tag', + attributes: [ + { type: 'attribute', name: 'name', value: name }, + { type: 'attribute', name: 'attrs', value: attrs }, + ], + }, + } as Token); + }, + + ontext(content: string | null | undefined) { + if (inCDATA) { + // ignore entirely while inside CDATA + return; + } + + // only accumulate text into the buffer if we're not under an ignored HTML element + if (typeof content === 'string') { + appendText(content); + } + }, + + // when an HTML tag closes... + onclosetag(name) { + // process any buffered text to be treated as a text node inside the currently closing HTML tag + processTextBuffer(); + + // push an 'html-tag' 'tag_close' Markdoc node instance for the currently closing HTML tag onto the resulting Token stack + output.push({ + type: 'tag_close', + nesting: -1, + meta: { + tag: 'html-tag', + attributes: [{ type: 'attribute', name: 'name', value: name }], + }, + } as Token); + }, + }, + { + decodeEntities: false, + recognizeCDATA: true, + recognizeSelfClosing: true, + }, + ); + + // for every detected token... + for (const token of tokens) { + // if it was an HTML token, write the HTML text into the HTML parser + if (token.type.startsWith('html')) { + // as the parser encounters opening/closing HTML tags, it will push Markdoc Tag nodes into the output stack + parser.write(token.content); + + // continue loop... IMPORTANT! we're throwing away the original 'html' tokens here (raw HTML strings), since the parser is inserting new ones based on the parsed HTML + continue; + } + + // process any child content for HTML + if (token.type === 'inline') { + if (token.children) { + token.children = htmlTokenTransform(tokenizer, token.children); + } + } + + // not an HTML Token, preserve it at the current stack location + output.push(token); + } + + // process any remaining buffered text + processTextBuffer(); + + // + // post-process the current levels output Token[] array to un-wind this pattern: + // + // [ + // { type: tag_open, meta.tag: html-tag }, + // { type: paragraph_open }, + // { type: inline, children [...] }, + // { type: paragraph_close }, + // { type: tag_close, meta.tag: html-tag } + // ] + // + // the paragraph_open, inline, paragraph_close triplet needs to be replaced by the children of the inline node + // + // this is extra, unwanted paragraph wrapping unfortunately introduced by markdown-it during processing w/ HTML enabled + // + + mutateAndCollapseExtraParagraphsUnderHtml(output); + + return output; +} + +function mutateAndCollapseExtraParagraphsUnderHtml(tokens: Token[]): void { + let done = false; + + while (!done) { + const idx = findExtraParagraphUnderHtml(tokens); + if (typeof idx === 'number') { + // mutate + + const actualChildTokens = tokens[idx + 2].children ?? []; + + tokens.splice(idx, 5, ...actualChildTokens); + } else { + done = true; + } + } +} + +/** + * + * @param token + * @returns + */ +function findExtraParagraphUnderHtml(tokens: Token[]): number | null { + if (tokens.length < 5) { + return null; + } + + for (let i = 0; i < tokens.length; i++) { + const last = i + 4; + if (last > tokens.length - 1) { + break; // early exit, no more possible 5-long slices to search + } + + const slice = tokens.slice(i, last + 1); + const isMatch = isExtraParagraphPatternMatch(slice); + if (isMatch) { + return i; + } + } + + return null; +} + +function isExtraParagraphPatternMatch(slice: Token[]): boolean { + const match = + isHtmlTagOpen(slice[0]) && + isParagraphOpen(slice[1]) && + isInline(slice[2]) && + isParagraphClose(slice[3]) && + isHtmlTagClose(slice[4]); + return match; +} + +function isHtmlTagOpen(token: Token): boolean { + return token.type === 'tag_open' && token.meta && token.meta.tag === 'html-tag'; +} + +function isHtmlTagClose(token: Token): boolean { + return token.type === 'tag_close' && token.meta && token.meta.tag === 'html-tag'; +} + +function isParagraphOpen(token: Token): boolean { + return token.type === 'paragraph_open'; +} + +function isParagraphClose(token: Token): boolean { + return token.type === 'paragraph_close'; +} + +function isInline(token: Token): boolean { + return token.type === 'inline'; +} diff --git a/packages/integrations/markdoc/src/index.ts b/packages/integrations/markdoc/src/index.ts new file mode 100644 index 000000000..d328d4a8d --- /dev/null +++ b/packages/integrations/markdoc/src/index.ts @@ -0,0 +1,49 @@ +import type { AstroConfig, AstroIntegration, ContentEntryType, HookParameters } from 'astro'; +import { getContentEntryType } from './content-entry-type.js'; +import { + type MarkdocConfigResult, + SUPPORTED_MARKDOC_CONFIG_FILES, + loadMarkdocConfig, +} from './load-config.js'; +import type { MarkdocIntegrationOptions } from './options.js'; + +type SetupHookParams = HookParameters<'astro:config:setup'> & { + // `contentEntryType` is not a public API + // Add type defs here + addContentEntryType: (contentEntryType: ContentEntryType) => void; +}; + +export default function markdocIntegration(options?: MarkdocIntegrationOptions): AstroIntegration { + let markdocConfigResult: MarkdocConfigResult | undefined; + let astroConfig: AstroConfig; + return { + name: '@astrojs/markdoc', + hooks: { + 'astro:config:setup': async (params) => { + const { updateConfig, addContentEntryType } = params as SetupHookParams; + astroConfig = params.config; + + markdocConfigResult = await loadMarkdocConfig(astroConfig); + + addContentEntryType( + await getContentEntryType({ markdocConfigResult, astroConfig, options }), + ); + + updateConfig({ + vite: { + ssr: { + external: ['@astrojs/markdoc/prism', '@astrojs/markdoc/shiki'], + }, + }, + }); + }, + 'astro:server:setup': async ({ server }) => { + server.watcher.on('all', (_event, entry) => { + if (SUPPORTED_MARKDOC_CONFIG_FILES.some((f) => entry.endsWith(f))) { + server.restart(); + } + }); + }, + }, + }; +} diff --git a/packages/integrations/markdoc/src/load-config.ts b/packages/integrations/markdoc/src/load-config.ts new file mode 100644 index 000000000..ce181e604 --- /dev/null +++ b/packages/integrations/markdoc/src/load-config.ts @@ -0,0 +1,118 @@ +import * as fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import type { AstroConfig } from 'astro'; +import { build as esbuild } from 'esbuild'; +import type { AstroMarkdocConfig } from './config.js'; +import { MarkdocError } from './utils.js'; + +export const SUPPORTED_MARKDOC_CONFIG_FILES = [ + 'markdoc.config.js', + 'markdoc.config.mjs', + 'markdoc.config.mts', + 'markdoc.config.ts', +]; + +export type MarkdocConfigResult = { + config: AstroMarkdocConfig; + fileUrl: URL; +}; + +export async function loadMarkdocConfig( + astroConfig: Pick<AstroConfig, 'root'>, +): Promise<MarkdocConfigResult | undefined> { + let markdocConfigUrl: URL | undefined; + for (const filename of SUPPORTED_MARKDOC_CONFIG_FILES) { + const filePath = new URL(filename, astroConfig.root); + if (!fs.existsSync(filePath)) continue; + + markdocConfigUrl = filePath; + break; + } + if (!markdocConfigUrl) return; + + const { code } = await bundleConfigFile({ + markdocConfigUrl, + astroConfig, + }); + const config: AstroMarkdocConfig = await loadConfigFromBundledFile(astroConfig.root, code); + + return { + config, + fileUrl: markdocConfigUrl, + }; +} + +/** + * Bundle config file to support `.ts` files. + * Simplified fork from Vite's `bundleConfigFile` function: + * @see https://github.com/vitejs/vite/blob/main/packages/vite/src/node/config.ts#L961 + */ +async function bundleConfigFile({ + markdocConfigUrl, + astroConfig, +}: { + markdocConfigUrl: URL; + astroConfig: Pick<AstroConfig, 'root'>; +}): Promise<{ code: string; dependencies: string[] }> { + let markdocError: MarkdocError | undefined; + + const result = await esbuild({ + absWorkingDir: fileURLToPath(astroConfig.root), + entryPoints: [fileURLToPath(markdocConfigUrl)], + outfile: 'out.js', + write: false, + target: ['node16'], + platform: 'node', + packages: 'external', + bundle: true, + format: 'esm', + sourcemap: 'inline', + metafile: true, + plugins: [ + { + name: 'stub-astro-imports', + setup(build) { + build.onResolve({ filter: /.*\.astro$/ }, () => { + // Avoid throwing within esbuild. + // This swallows the `hint` and blows up the stacktrace. + markdocError = new MarkdocError({ + message: '`.astro` files are no longer supported in the Markdoc config.', + hint: 'Use the `component()` utility to specify a component path instead. See https://docs.astro.build/en/guides/integrations-guide/markdoc/', + }); + return { + // Stub with an unused default export. + path: 'data:text/javascript,export default true', + external: true, + }; + }); + }, + }, + ], + }); + if (markdocError) throw markdocError; + const { text } = result.outputFiles[0]; + return { + code: text, + dependencies: result.metafile ? Object.keys(result.metafile.inputs) : [], + }; +} + +/** + * Forked from Vite config loader, replacing CJS-based path concat + * with ESM only + * @see https://github.com/vitejs/vite/blob/main/packages/vite/src/node/config.ts#L1074 + */ +async function loadConfigFromBundledFile(root: URL, code: string): Promise<AstroMarkdocConfig> { + // Write it to disk, load it with native Node ESM, then delete the file. + const tmpFileUrl = new URL(`markdoc.config.timestamp-${Date.now()}.mjs`, root); + fs.writeFileSync(tmpFileUrl, code); + try { + return (await import(tmpFileUrl.pathname)).default; + } finally { + try { + fs.unlinkSync(tmpFileUrl); + } catch { + // already removed if this function is called twice simultaneously + } + } +} diff --git a/packages/integrations/markdoc/src/options.ts b/packages/integrations/markdoc/src/options.ts new file mode 100644 index 000000000..abaeb5a96 --- /dev/null +++ b/packages/integrations/markdoc/src/options.ts @@ -0,0 +1,5 @@ +export interface MarkdocIntegrationOptions { + allowHTML?: boolean; + ignoreIndentation?: boolean; + typographer?: boolean; +} diff --git a/packages/integrations/markdoc/src/runtime-assets-config.ts b/packages/integrations/markdoc/src/runtime-assets-config.ts new file mode 100644 index 000000000..0211c1381 --- /dev/null +++ b/packages/integrations/markdoc/src/runtime-assets-config.ts @@ -0,0 +1,26 @@ +//@ts-expect-error Cannot find module 'astro:assets' or its corresponding type declarations. +import { Image } from 'astro:assets'; +import type { Config as MarkdocConfig } from '@markdoc/markdoc'; +import Markdoc from '@markdoc/markdoc'; + +export const assetsConfig: MarkdocConfig = { + nodes: { + image: { + attributes: { + ...Markdoc.nodes.image.attributes, + __optimizedSrc: { type: 'Object' }, + }, + transform(node, config) { + const attributes = node.transformAttributes(config); + const children = node.transformChildren(config); + + if (node.type === 'image' && '__optimizedSrc' in node.attributes) { + const { __optimizedSrc, ...rest } = node.attributes; + return new Markdoc.Tag(Image, { ...rest, src: __optimizedSrc }, children); + } else { + return new Markdoc.Tag('img', attributes, children); + } + }, + }, + }, +}; diff --git a/packages/integrations/markdoc/src/runtime.ts b/packages/integrations/markdoc/src/runtime.ts new file mode 100644 index 000000000..f62bcec1a --- /dev/null +++ b/packages/integrations/markdoc/src/runtime.ts @@ -0,0 +1,213 @@ +import type { MarkdownHeading } from '@astrojs/markdown-remark'; +import Markdoc, { + type ConfigType, + type Node, + type NodeType, + type RenderableTreeNode, +} from '@markdoc/markdoc'; +import type { AstroInstance } from 'astro'; +import { createComponent, renderComponent } from 'astro/runtime/server/index.js'; +import type { AstroMarkdocConfig } from './config.js'; +import { setupHeadingConfig } from './heading-ids.js'; +import { htmlTag } from './html/tagdefs/html.tag.js'; +import type { MarkdocIntegrationOptions } from './options.js'; +/** + * Merge user config with default config and set up context (ex. heading ID slugger) + * Called on each file's individual transform. + * TODO: virtual module to merge configs per-build instead of per-file? + */ +export async function setupConfig( + userConfig: AstroMarkdocConfig = {}, + options: MarkdocIntegrationOptions | undefined, +): Promise<MergedConfig> { + let defaultConfig: AstroMarkdocConfig = setupHeadingConfig(); + + if (userConfig.extends) { + for (let extension of userConfig.extends) { + if (extension instanceof Promise) { + extension = await extension; + } + + defaultConfig = mergeConfig(defaultConfig, extension); + } + } + + let merged = mergeConfig(defaultConfig, userConfig); + + if (options?.allowHTML) { + merged = mergeConfig(merged, HTML_CONFIG); + } + + return merged; +} + +/** Used for synchronous `getHeadings()` function */ +export function setupConfigSync( + userConfig: AstroMarkdocConfig = {}, + options: MarkdocIntegrationOptions | undefined, +): MergedConfig { + const defaultConfig: AstroMarkdocConfig = setupHeadingConfig(); + + let merged = mergeConfig(defaultConfig, userConfig); + + if (options?.allowHTML) { + merged = mergeConfig(merged, HTML_CONFIG); + } + + return merged; +} + +type MergedConfig = Required<Omit<AstroMarkdocConfig, 'extends'>>; + +/** Merge function from `@markdoc/markdoc` internals */ +export function mergeConfig( + configA: AstroMarkdocConfig, + configB: AstroMarkdocConfig, +): MergedConfig { + return { + ...configA, + ...configB, + ctx: { + ...configA.ctx, + ...configB.ctx, + }, + tags: { + ...configA.tags, + ...configB.tags, + }, + nodes: { + ...configA.nodes, + ...configB.nodes, + }, + functions: { + ...configA.functions, + ...configB.functions, + }, + variables: { + ...configA.variables, + ...configB.variables, + }, + partials: { + ...configA.partials, + ...configB.partials, + }, + validation: { + ...configA.validation, + ...configB.validation, + }, + }; +} + +export function resolveComponentImports( + markdocConfig: Required<Pick<AstroMarkdocConfig, 'tags' | 'nodes'>>, + tagComponentMap: Record<string, AstroInstance['default']>, + nodeComponentMap: Record<NodeType, AstroInstance['default']>, +) { + for (const [tag, render] of Object.entries(tagComponentMap)) { + const config = markdocConfig.tags[tag]; + if (config) config.render = render; + } + for (const [node, render] of Object.entries(nodeComponentMap)) { + const config = markdocConfig.nodes[node as NodeType]; + if (config) config.render = render; + } + return markdocConfig; +} + +/** + * Get text content as a string from a Markdoc transform AST + */ +export function getTextContent(childNodes: RenderableTreeNode[]): string { + let text = ''; + for (const node of childNodes) { + if (typeof node === 'string' || typeof node === 'number') { + text += node; + } else if (typeof node === 'object' && Markdoc.Tag.isTag(node)) { + text += getTextContent(node.children); + } + } + return text; +} + +const headingLevels = [1, 2, 3, 4, 5, 6] as const; + +/** + * Collect headings from Markdoc transform AST + * for `headings` result on `render()` return value + */ +export function collectHeadings( + children: RenderableTreeNode[], + collectedHeadings: MarkdownHeading[], +) { + for (const node of children) { + if (typeof node !== 'object' || !Markdoc.Tag.isTag(node)) continue; + + if (node.attributes.__collectHeading === true && typeof node.attributes.level === 'number') { + collectedHeadings.push({ + slug: node.attributes.id, + depth: node.attributes.level, + text: getTextContent(node.children), + }); + continue; + } + + for (const level of headingLevels) { + if (node.name === 'h' + level) { + collectedHeadings.push({ + slug: node.attributes.id, + depth: level, + text: getTextContent(node.children), + }); + } + } + collectHeadings(node.children, collectedHeadings); + } +} + +export function createGetHeadings( + stringifiedAst: string, + userConfig: AstroMarkdocConfig, + options: MarkdocIntegrationOptions | undefined, +) { + return function getHeadings() { + /* Yes, we are transforming twice (once from `getHeadings()` and again from <Content /> in case of variables). + TODO: propose new `render()` API to allow Markdoc variable passing to `render()` itself, + instead of the Content component. Would remove double-transform and unlock variable resolution in heading slugs. */ + const config = setupConfigSync(userConfig, options); + const ast = Markdoc.Ast.fromJSON(stringifiedAst); + const content = Markdoc.transform(ast as Node, config as ConfigType); + let collectedHeadings: MarkdownHeading[] = []; + collectHeadings(Array.isArray(content) ? content : [content], collectedHeadings); + return collectedHeadings; + }; +} + +export function createContentComponent( + Renderer: AstroInstance['default'], + stringifiedAst: string, + userConfig: AstroMarkdocConfig, + options: MarkdocIntegrationOptions | undefined, + tagComponentMap: Record<string, AstroInstance['default']>, + nodeComponentMap: Record<NodeType, AstroInstance['default']>, +) { + return createComponent({ + async factory(result: any, props: Record<string, any>) { + const withVariables = mergeConfig(userConfig, { variables: props }); + const config = resolveComponentImports( + await setupConfig(withVariables, options), + tagComponentMap, + nodeComponentMap, + ); + + return renderComponent(result, Renderer.name, Renderer, { stringifiedAst, config }, {}); + }, + propagation: 'self', + } as any); +} + +// statically define a partial MarkdocConfig which registers the required "html-tag" Markdoc tag when the "allowHTML" feature is enabled +const HTML_CONFIG: AstroMarkdocConfig = { + tags: { + 'html-tag': htmlTag, + }, +}; diff --git a/packages/integrations/markdoc/src/tokenizer.ts b/packages/integrations/markdoc/src/tokenizer.ts new file mode 100644 index 000000000..1f5b1de28 --- /dev/null +++ b/packages/integrations/markdoc/src/tokenizer.ts @@ -0,0 +1,44 @@ +import type { Tokenizer } from '@markdoc/markdoc'; +import Markdoc from '@markdoc/markdoc'; +import type { MarkdocIntegrationOptions } from './options.js'; + +type TokenizerOptions = ConstructorParameters<typeof Tokenizer>[0]; + +export function getMarkdocTokenizer(options: MarkdocIntegrationOptions | undefined): Tokenizer { + const key = cacheKey(options); + + if (!_cachedMarkdocTokenizers[key]) { + const tokenizerOptions: TokenizerOptions = { + // Strip <!-- comments --> from rendered output + // Without this, they're rendered as strings! + allowComments: true, + }; + + if (options?.allowHTML) { + // allow indentation for Markdoc tags that are interleaved inside HTML block elements + tokenizerOptions.allowIndentation = true; + // enable HTML token detection in markdown-it + tokenizerOptions.html = true; + } + if (options?.ignoreIndentation) { + // allow indentation so nested Markdoc tags can be formatted for better readability + tokenizerOptions.allowIndentation = true; + } + if (options?.typographer) { + // enable typographer to convert straight quotes to curly quotes, etc. + tokenizerOptions.typographer = options.typographer; + } + + _cachedMarkdocTokenizers[key] = new Markdoc.Tokenizer(tokenizerOptions); + } + + return _cachedMarkdocTokenizers[key]; +} + +// create this on-demand when needed since it relies on the runtime MarkdocIntegrationOptions and may change during +// the life of module in certain scenarios (unit tests, etc.) +let _cachedMarkdocTokenizers: Record<string, Tokenizer> = {}; + +function cacheKey(options: MarkdocIntegrationOptions | undefined): string { + return JSON.stringify(options); +} diff --git a/packages/integrations/markdoc/src/utils.ts b/packages/integrations/markdoc/src/utils.ts new file mode 100644 index 000000000..3f2aed633 --- /dev/null +++ b/packages/integrations/markdoc/src/utils.ts @@ -0,0 +1,82 @@ +import type { ComponentConfig } from './config.js'; + +/** + * Matches AstroError object with types like error codes stubbed out + * @see 'astro/src/core/errors/errors.ts' + */ +export class MarkdocError extends Error { + public loc: ErrorLocation | undefined; + public title: string | undefined; + public hint: string | undefined; + public frame: string | undefined; + + type = 'MarkdocError'; + + constructor(props: ErrorProperties, ...params: any) { + super(...params); + + const { title = 'MarkdocError', message, stack, location, hint, frame } = props; + + this.title = title; + if (message) this.message = message; + // Only set this if we actually have a stack passed, otherwise uses Error's + this.stack = stack ? stack : this.stack; + this.loc = location; + this.hint = hint; + this.frame = frame; + } +} + +interface ErrorLocation { + file?: string; + line?: number; + column?: number; +} + +interface ErrorProperties { + code?: number; + title?: string; + name?: string; + message?: string; + location?: ErrorLocation; + hint?: string; + stack?: string; + frame?: string; +} + +/** + * @see 'astro/src/core/path.ts' + */ +export function prependForwardSlash(str: string) { + return str[0] === '/' ? str : '/' + str; +} + +export function isValidUrl(str: string): boolean { + try { + new URL(str); + return true; + } catch { + return false; + } +} + +/** + * Identifies Astro components with propagated assets + * @see 'packages/astro/src/content/consts.ts' + */ +export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets'; + +/** + * @see 'packages/astro/src/content/utils.ts' + */ +export function hasContentFlag(viteId: string, flag: string): boolean { + const flags = new URLSearchParams(viteId.split('?')[1] ?? ''); + return flags.has(flag); +} + +/** Identifier for components imports passed as `tags` or `nodes` configuration. */ +export const componentConfigSymbol = Symbol.for('@astrojs/markdoc/component-config'); + +export function isComponentConfig(value: unknown): value is ComponentConfig { + return typeof value === 'object' && value !== null && componentConfigSymbol in value; +} |