From 3639190b4e1b4c97836d448fa80a58aa45c823a7 Mon Sep 17 00:00:00 2001 From: Drew Powers <1369770+drwpow@users.noreply.github.com> Date: Mon, 12 Apr 2021 17:21:29 -0600 Subject: Renaming to import.meta.fetchContent (#70) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Change to import.meta.glob() Change of plans—maintain parity with Snowpack and Vite because our Collections API will use a different interface * Get basic pagination working * Get params working * Rename to import.meta.fetchContent * Upgrade to fdir --- src/compiler/codegen/index.ts | 656 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 656 insertions(+) create mode 100644 src/compiler/codegen/index.ts (limited to 'src/compiler/codegen/index.ts') diff --git a/src/compiler/codegen/index.ts b/src/compiler/codegen/index.ts new file mode 100644 index 000000000..d2ac96702 --- /dev/null +++ b/src/compiler/codegen/index.ts @@ -0,0 +1,656 @@ +import type { CompileOptions } from '../../@types/compiler'; +import type { AstroConfig, ValidExtensionPlugins } from '../../@types/astro'; +import type { Ast, Script, Style, TemplateNode } from '../../parser/interfaces'; +import type { TransformResult } from '../../@types/astro'; + +import eslexer from 'es-module-lexer'; +import esbuild from 'esbuild'; +import path from 'path'; +import { walk } from 'estree-walker'; +import _babelGenerator from '@babel/generator'; +import babelParser from '@babel/parser'; +import * as babelTraverse from '@babel/traverse'; +import { ImportDeclaration, ExportNamedDeclaration, VariableDeclarator, Identifier } from '@babel/types'; +import { warn } from '../../logger.js'; +import { fetchContent } from './content.js'; +import { isImportMetaDeclaration } from './utils.js'; +import { yellow } from 'kleur/colors'; + +const traverse: typeof babelTraverse.default = (babelTraverse.default as any).default; +const babelGenerator: typeof _babelGenerator = + // @ts-ignore + _babelGenerator.default; +const { transformSync } = esbuild; + +interface Attribute { + start: number; + end: number; + type: 'Attribute'; + name: string; + value: TemplateNode[] | boolean; +} + +interface CodeGenOptions { + compileOptions: CompileOptions; + filename: string; + fileID: string; +} + +/** Format Astro internal import URL */ +function internalImport(internalPath: string) { + return `/_astro_internal/${internalPath}`; +} + +/** Retrieve attributes from TemplateNode */ +function getAttributes(attrs: Attribute[]): Record { + let result: Record = {}; + for (const attr of attrs) { + if (attr.value === true) { + result[attr.name] = JSON.stringify(attr.value); + continue; + } + if (attr.value === false || attr.value === undefined) { + // note: attr.value shouldn’t be `undefined`, but a bad transform would cause a compile error here, so prevent that + continue; + } + if (attr.value.length > 1) { + result[attr.name] = + '(' + + attr.value + .map((v: TemplateNode) => { + if (v.content) { + return v.content; + } else { + return JSON.stringify(getTextFromAttribute(v)); + } + }) + .join('+') + + ')'; + continue; + } + const val = attr.value[0]; + if (!val) { + result[attr.name] = '(' + val + ')'; + continue; + } + switch (val.type) { + case 'MustacheTag': { + result[attr.name] = '(' + val.expression.codeStart + ')'; + continue; + } + case 'Text': + result[attr.name] = JSON.stringify(getTextFromAttribute(val)); + continue; + default: + throw new Error(`UNKNOWN: ${val.type}`); + } + } + return result; +} + +/** Get value from a TemplateNode Attribute (text attributes only!) */ +function getTextFromAttribute(attr: any): string { + switch (attr.type) { + case 'Text': { + if (attr.raw !== undefined) { + return attr.raw; + } + if (attr.data !== undefined) { + return attr.data; + } + break; + } + case 'MustacheTag': { + return attr.expression.codeStart; + } + } + throw new Error(`Unknown attribute type ${attr.type}`); +} + +/** Convert TemplateNode attributes to string */ +function generateAttributes(attrs: Record): string { + let result = '{'; + for (const [key, val] of Object.entries(attrs)) { + result += JSON.stringify(key) + ':' + val + ','; + } + return result + '}'; +} + +interface ComponentInfo { + type: string; + url: string; + plugin: string | undefined; +} + +const defaultExtensions: Readonly> = { + '.astro': 'astro', + '.jsx': 'react', + '.vue': 'vue', + '.svelte': 'svelte', +}; + +type DynamicImportMap = Map<'vue' | 'react' | 'react-dom' | 'preact', string>; + +interface GetComponentWrapperOptions { + filename: string; + astroConfig: AstroConfig; + dynamicImports: DynamicImportMap; +} + +/** Generate Astro-friendly component import */ +function getComponentWrapper(_name: string, { type, plugin, url }: ComponentInfo, opts: GetComponentWrapperOptions) { + const { astroConfig, dynamicImports, filename } = opts; + const { astroRoot } = astroConfig; + const [name, kind] = _name.split(':'); + const currFileUrl = new URL(`file://${filename}`); + + if (!plugin) { + throw new Error(`No supported plugin found for extension ${type}`); + } + + const getComponentUrl = (ext = '.js') => { + const outUrl = new URL(url, currFileUrl); + return '/_astro/' + path.posix.relative(astroRoot.pathname, outUrl.pathname).replace(/\.[^.]+$/, ext); + }; + + switch (plugin) { + case 'astro': { + if (kind) { + throw new Error(`Astro does not support :${kind}`); + } + return { + wrapper: name, + wrapperImport: ``, + }; + } + case 'preact': { + if (['load', 'idle', 'visible'].includes(kind)) { + return { + wrapper: `__preact_${kind}(${name}, ${JSON.stringify({ + componentUrl: getComponentUrl(), + componentExport: 'default', + frameworkUrls: { + preact: dynamicImports.get('preact'), + }, + })})`, + wrapperImport: `import {__preact_${kind}} from '${internalImport('render/preact.js')}';`, + }; + } + + return { + wrapper: `__preact_static(${name})`, + wrapperImport: `import {__preact_static} from '${internalImport('render/preact.js')}';`, + }; + } + case 'react': { + if (['load', 'idle', 'visible'].includes(kind)) { + return { + wrapper: `__react_${kind}(${name}, ${JSON.stringify({ + componentUrl: getComponentUrl(), + componentExport: 'default', + frameworkUrls: { + react: dynamicImports.get('react'), + 'react-dom': dynamicImports.get('react-dom'), + }, + })})`, + wrapperImport: `import {__react_${kind}} from '${internalImport('render/react.js')}';`, + }; + } + + return { + wrapper: `__react_static(${name})`, + wrapperImport: `import {__react_static} from '${internalImport('render/react.js')}';`, + }; + } + case 'svelte': { + if (['load', 'idle', 'visible'].includes(kind)) { + return { + wrapper: `__svelte_${kind}(${name}, ${JSON.stringify({ + componentUrl: getComponentUrl('.svelte.js'), + componentExport: 'default', + })})`, + wrapperImport: `import {__svelte_${kind}} from '${internalImport('render/svelte.js')}';`, + }; + } + + return { + wrapper: `__svelte_static(${name})`, + wrapperImport: `import {__svelte_static} from '${internalImport('render/svelte.js')}';`, + }; + } + case 'vue': { + if (['load', 'idle', 'visible'].includes(kind)) { + return { + wrapper: `__vue_${kind}(${name}, ${JSON.stringify({ + componentUrl: getComponentUrl('.vue.js'), + componentExport: 'default', + frameworkUrls: { + vue: dynamicImports.get('vue'), + }, + })})`, + wrapperImport: `import {__vue_${kind}} from '${internalImport('render/vue.js')}';`, + }; + } + + return { + wrapper: `__vue_static(${name})`, + wrapperImport: `import {__vue_static} from '${internalImport('render/vue.js')}';`, + }; + } + default: { + throw new Error(`Unknown component type`); + } + } +} + +/** Evaluate expression (safely) */ +function compileExpressionSafe(raw: string): string { + let { code } = transformSync(raw, { + loader: 'tsx', + jsxFactory: 'h', + jsxFragment: 'Fragment', + charset: 'utf8', + }); + return code; +} + +/** Build dependency map of dynamic component runtime frameworks */ +async function acquireDynamicComponentImports(plugins: Set, resolve: (s: string) => Promise): Promise { + const importMap: DynamicImportMap = new Map(); + for (let plugin of plugins) { + switch (plugin) { + case 'vue': { + importMap.set('vue', await resolve('vue')); + break; + } + case 'react': { + importMap.set('react', await resolve('react')); + importMap.set('react-dom', await resolve('react-dom')); + break; + } + case 'preact': { + importMap.set('preact', await resolve('preact')); + break; + } + } + } + return importMap; +} + +type Components = Record; + +interface CompileResult { + script: string; + componentPlugins: Set; + createCollection?: string; +} + +interface CodegenState { + filename: string; + components: Components; + css: string[]; + importExportStatements: Set; + dynamicImports: DynamicImportMap; +} + +/** Compile/prepare Astro frontmatter scripts */ +function compileModule(module: Script, state: CodegenState, compileOptions: CompileOptions): CompileResult { + const { extensions = defaultExtensions } = compileOptions; + + const componentImports: ImportDeclaration[] = []; + const componentProps: VariableDeclarator[] = []; + const componentExports: ExportNamedDeclaration[] = []; + + const contentImports = new Map(); + + let script = ''; + let propsStatement = ''; + let contentCode = ''; // code for handling import.meta.fetchContent(), if any; + let createCollection = ''; // function for executing collection + const componentPlugins = new Set(); + + if (module) { + const program = babelParser.parse(module.content, { + sourceType: 'module', + plugins: ['jsx', 'typescript', 'topLevelAwait'], + }).program; + + const { body } = program; + let i = body.length; + while (--i >= 0) { + const node = body[i]; + switch (node.type) { + case 'ExportNamedDeclaration': { + if (!node.declaration) break; + // const replacement = extract_exports(node); + + if (node.declaration.type === 'VariableDeclaration') { + // case 1: prop (export let title) + + const declaration = node.declaration.declarations[0]; + if ((declaration.id as Identifier).name === '__layout' || (declaration.id as Identifier).name === '__content') { + componentExports.push(node); + } else { + componentProps.push(declaration); + } + body.splice(i, 1); + } else if (node.declaration.type === 'FunctionDeclaration') { + // case 2: createCollection (export async function) + if (!node.declaration.id || node.declaration.id.name !== 'createCollection') break; + createCollection = module.content.substring(node.declaration.start || 0, node.declaration.end || 0); + + // remove node + body.splice(i, 1); + } + break; + } + case 'FunctionDeclaration': { + break; + } + case 'ImportDeclaration': { + componentImports.push(node); + body.splice(i, 1); // remove node + break; + } + case 'VariableDeclaration': { + for (const declaration of node.declarations) { + // only select import.meta.fetchContent() calls here. this utility filters those out for us. + if (!isImportMetaDeclaration(declaration, 'fetchContent')) continue; + + // remove node + body.splice(i, 1); + + // a bit of munging + let { id, init } = declaration; + if (!id || !init || id.type !== 'Identifier') continue; + if (init.type === 'AwaitExpression') { + init = init.argument; + const shortname = path.relative(compileOptions.astroConfig.projectRoot.pathname, state.filename); + warn(compileOptions.logging, shortname, yellow('awaiting import.meta.fetchContent() not necessary')); + } + if (init.type !== 'CallExpression') continue; + + // gather data + const namespace = id.name; + + if ((init as any).arguments[0].type !== 'StringLiteral') { + throw new Error(`[import.meta.fetchContent] Only string literals allowed, ex: \`import.meta.fetchContent('./post/*.md')\`\n ${state.filename}`); + } + const spec = (init as any).arguments[0].value; + if (typeof spec === 'string') contentImports.set(namespace, { spec, declarator: node.kind }); + } + break; + } + } + } + + for (const componentImport of componentImports) { + const importUrl = componentImport.source.value; + const componentType = path.posix.extname(importUrl); + const specifier = componentImport.specifiers[0]; + if (!specifier) continue; // this is unused + // set componentName to default import if used (user), or use filename if no default import (mostly internal use) + const componentName = specifier.type === 'ImportDefaultSpecifier' ? specifier.local.name : path.posix.basename(importUrl, componentType); + const plugin = extensions[componentType] || defaultExtensions[componentType]; + state.components[componentName] = { + type: componentType, + plugin, + url: importUrl, + }; + if (plugin) { + componentPlugins.add(plugin); + } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + state.importExportStatements.add(module.content.slice(componentImport.start!, componentImport.end!)); + } + for (const componentImport of componentExports) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + state.importExportStatements.add(module.content.slice(componentImport.start!, componentImport.end!)); + } + + if (componentProps.length > 0) { + propsStatement = 'let {'; + for (const componentExport of componentProps) { + propsStatement += `${(componentExport.id as Identifier).name}`; + if (componentExport.init) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + propsStatement += `= ${babelGenerator(componentExport.init!).code}`; + } + propsStatement += `,`; + } + propsStatement += `} = props;\n`; + } + + // handle createCollection, if any + if (createCollection) { + // TODO: improve this? while transforming in-place isn’t great, this happens at most once per-route + const ast = babelParser.parse(createCollection, { + sourceType: 'module', + }); + traverse(ast, { + enter({ node }) { + switch (node.type) { + case 'VariableDeclaration': { + for (const declaration of node.declarations) { + // only select import.meta.collection() calls here. this utility filters those out for us. + if (!isImportMetaDeclaration(declaration, 'fetchContent')) continue; + + // a bit of munging + let { id, init } = declaration; + if (!id || !init || id.type !== 'Identifier') continue; + if (init.type === 'AwaitExpression') { + init = init.argument; + const shortname = path.relative(compileOptions.astroConfig.projectRoot.pathname, state.filename); + warn(compileOptions.logging, shortname, yellow('awaiting import.meta.fetchContent() not necessary')); + } + if (init.type !== 'CallExpression') continue; + + // gather data + const namespace = id.name; + + if ((init as any).arguments[0].type !== 'StringLiteral') { + throw new Error(`[import.meta.fetchContent] Only string literals allowed, ex: \`import.meta.fetchContent('./post/*.md')\`\n ${state.filename}`); + } + const spec = (init as any).arguments[0].value; + if (typeof spec !== 'string') break; + + const globResult = fetchContent(spec, { namespace, filename: state.filename }); + + let imports = ''; + for (const importStatement of globResult.imports) { + imports += importStatement + '\n'; + } + + createCollection = + imports + '\n\nexport ' + createCollection.substring(0, declaration.start || 0) + globResult.code + createCollection.substring(declaration.end || 0); + } + break; + } + } + }, + }); + } + + // import.meta.fetchContent() + for (const [namespace, { declarator, spec }] of contentImports.entries()) { + const globResult = fetchContent(spec, { namespace, filename: state.filename }); + for (const importStatement of globResult.imports) { + state.importExportStatements.add(importStatement); + } + contentCode += globResult.code; + } + + script = propsStatement + contentCode + babelGenerator(program).code; + } + + return { + script, + componentPlugins, + createCollection: createCollection || undefined, + }; +} + +/** Compile styles */ +function compileCss(style: Style, state: CodegenState) { + walk(style, { + enter(node: TemplateNode) { + if (node.type === 'Style') { + state.css.push(node.content.styles); // if multiple