diff options
Diffstat (limited to 'packages/astro/src')
48 files changed, 4380 insertions, 0 deletions
diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts new file mode 100644 index 000000000..049105970 --- /dev/null +++ b/packages/astro/src/@types/astro.ts @@ -0,0 +1,132 @@ +export interface AstroConfigRaw { + dist: string; + projectRoot: string; + astroRoot: string; + public: string; + jsx?: string; +} + +export type ValidExtensionPlugins = 'astro' | 'react' | 'preact' | 'svelte' | 'vue'; + +export interface AstroConfig { + dist: string; + projectRoot: URL; + astroRoot: URL; + public: URL; + extensions?: Record<string, ValidExtensionPlugins>; + /** Options specific to `astro build` */ + buildOptions: { + /** Your public domain, e.g.: https://my-site.dev/. Used to generate sitemaps and canonical URLs. */ + site?: string; + /** Generate sitemap (set to "false" to disable) */ + sitemap: boolean; + }; + /** Options for the development server run with `astro dev`. */ + devOptions: { + /** The port to run the dev server on. */ + port: number; + projectRoot?: string; + }; +} + +export type AstroUserConfig = Omit<AstroConfig, 'buildOptions' | 'devOptions'> & { + buildOptions: { + sitemap: boolean; + }; + devOptions: { + port?: number; + projectRoot?: string; + }; +}; + +export interface JsxItem { + name: string; + jsx: string; +} + +export interface TransformResult { + script: string; + imports: string[]; + html: string; + css?: string; + /** If this page exports a collection, the JS to be executed as a string */ + createCollection?: string; +} + +export interface CompileResult { + result: TransformResult; + contents: string; + css?: string; +} + +export type RuntimeMode = 'development' | 'production'; + +export type Params = Record<string, string | number>; + +export interface CreateCollection<T = any> { + data: ({ params }: { params: Params }) => T[]; + routes?: Params[]; + /** tool for generating current page URL */ + permalink?: ({ params }: { params: Params }) => string; + /** page size */ + pageSize?: number; + /** Generate RSS feed from data() */ + rss?: CollectionRSS<T>; +} + +export interface CollectionRSS<T = any> { + /** (required) Title of the RSS Feed */ + title: string; + /** (required) Description of the RSS Feed */ + description: string; + /** Specify arbitrary metadata on opening <xml> tag */ + xmlns?: Record<string, string>; + /** Specify custom data in opening of file */ + customData?: string; + /** Return data about each item */ + item: ( + item: T + ) => { + /** (required) Title of item */ + title: string; + /** (required) Link to item */ + link: string; + /** Publication date of item */ + pubDate?: Date; + /** Item description */ + description?: string; + /** Append some other XML-valid data to this item */ + customData?: string; + }; +} + +export interface CollectionResult<T = any> { + /** result */ + data: T[]; + + /** metadata */ + /** the count of the first item on the page, starting from 0 */ + start: number; + /** the count of the last item on the page, starting from 0 */ + end: number; + /** total number of results */ + total: number; + page: { + /** the current page number, starting from 1 */ + current: number; + /** number of items per page (default: 25) */ + size: number; + /** number of last page */ + last: number; + }; + url: { + /** url of the current page */ + current: string; + /** url of the previous page (if there is one) */ + prev?: string; + /** url of the next page (if there is one) */ + next?: string; + }; + /** Matched parameters, if any */ + params: Params; +} diff --git a/packages/astro/src/@types/compiler.ts b/packages/astro/src/@types/compiler.ts new file mode 100644 index 000000000..7da0afaf2 --- /dev/null +++ b/packages/astro/src/@types/compiler.ts @@ -0,0 +1,10 @@ +import type { LogOptions } from '../logger'; +import type { AstroConfig, RuntimeMode, ValidExtensionPlugins } from './astro'; + +export interface CompileOptions { + logging: LogOptions; + resolvePackageUrl: (p: string) => Promise<string>; + astroConfig: AstroConfig; + extensions?: Record<string, ValidExtensionPlugins>; + mode: RuntimeMode; +} diff --git a/packages/astro/src/@types/estree-walker.d.ts b/packages/astro/src/@types/estree-walker.d.ts new file mode 100644 index 000000000..a3b7da859 --- /dev/null +++ b/packages/astro/src/@types/estree-walker.d.ts @@ -0,0 +1,25 @@ +import { BaseNode } from 'estree-walker'; + +declare module 'estree-walker' { + export function walk<T = BaseNode>( + ast: T, + { + enter, + leave, + }: { + enter?: (this: { skip: () => void; remove: () => void; replace: (node: T) => void }, node: T, parent: T, key: string, index: number) => void; + leave?: (this: { skip: () => void; remove: () => void; replace: (node: T) => void }, node: T, parent: T, key: string, index: number) => void; + } + ): T; + + export function asyncWalk<T = BaseNode>( + ast: T, + { + enter, + leave, + }: { + enter?: (this: { skip: () => void; remove: () => void; replace: (node: T) => void }, node: T, parent: T, key: string, index: number) => void; + leave?: (this: { skip: () => void; remove: () => void; replace: (node: T) => void }, node: T, parent: T, key: string, index: number) => void; + } + ): T; +} diff --git a/packages/astro/src/@types/micromark-extension-gfm.d.ts b/packages/astro/src/@types/micromark-extension-gfm.d.ts new file mode 100644 index 000000000..ebdfe3b3a --- /dev/null +++ b/packages/astro/src/@types/micromark-extension-gfm.d.ts @@ -0,0 +1,3 @@ +// TODO: add types (if helpful) +declare module 'micromark-extension-gfm'; +declare module 'micromark-extension-gfm/html.js'; diff --git a/packages/astro/src/@types/micromark.ts b/packages/astro/src/@types/micromark.ts new file mode 100644 index 000000000..9725aabb9 --- /dev/null +++ b/packages/astro/src/@types/micromark.ts @@ -0,0 +1,11 @@ +export interface MicromarkExtensionContext { + sliceSerialize(node: any): string; + raw(value: string): void; +} + +export type MicromarkExtensionCallback = (this: MicromarkExtensionContext, node: any) => void; + +export interface MicromarkExtension { + enter?: Record<string, MicromarkExtensionCallback>; + exit?: Record<string, MicromarkExtensionCallback>; +} diff --git a/packages/astro/src/@types/postcss-icss-keyframes.d.ts b/packages/astro/src/@types/postcss-icss-keyframes.d.ts new file mode 100644 index 000000000..14c330b6e --- /dev/null +++ b/packages/astro/src/@types/postcss-icss-keyframes.d.ts @@ -0,0 +1,5 @@ +declare module 'postcss-icss-keyframes' { + import type { Plugin } from 'postcss'; + + export default function (options: { generateScopedName(keyframesName: string, filepath: string, css: string): string }): Plugin; +} diff --git a/packages/astro/src/@types/renderer.ts b/packages/astro/src/@types/renderer.ts new file mode 100644 index 000000000..f89cb6664 --- /dev/null +++ b/packages/astro/src/@types/renderer.ts @@ -0,0 +1,37 @@ +import type { Component as VueComponent } from 'vue'; +import type { ComponentType as PreactComponent } from 'preact'; +import type { ComponentType as ReactComponent } from 'react'; +import type { SvelteComponent } from 'svelte'; + +export interface DynamicRenderContext { + componentUrl: string; + componentExport: string; + frameworkUrls: string; +} + +export interface ComponentRenderer<T> { + renderStatic: StaticRendererGenerator<T>; + jsxPragma?: (...args: any) => any; + jsxPragmaName?: string; + render(context: { root: string; Component: string; props: string; [key: string]: string }): string; + imports?: Record<string, string[]>; +} + +export interface ComponentContext { + 'data-astro-id': string; + root: string; +} + +export type SupportedComponentRenderer = + | ComponentRenderer<VueComponent> + | ComponentRenderer<PreactComponent> + | ComponentRenderer<ReactComponent> + | ComponentRenderer<SvelteComponent>; +export type StaticRenderer = (props: Record<string, any>, ...children: any[]) => Promise<string>; +export type StaticRendererGenerator<T = any> = (Component: T) => StaticRenderer; +export type DynamicRenderer = (props: Record<string, any>, ...children: any[]) => Promise<string>; +export type DynamicRendererContext<T = any> = (Component: T, renderContext: DynamicRenderContext) => DynamicRenderer; +export type DynamicRendererGenerator = ( + wrapperStart: string | ((context: ComponentContext) => string), + wrapperEnd: string | ((context: ComponentContext) => string) +) => DynamicRendererContext; diff --git a/packages/astro/src/@types/tailwind.d.ts b/packages/astro/src/@types/tailwind.d.ts new file mode 100644 index 000000000..d25eaae2f --- /dev/null +++ b/packages/astro/src/@types/tailwind.d.ts @@ -0,0 +1,2 @@ +// we shouldn‘t have this as a dependency for Astro, but we may dynamically import it if a user requests it, so let TS know about it +declare module 'tailwindcss'; diff --git a/packages/astro/src/@types/transformer.ts b/packages/astro/src/@types/transformer.ts new file mode 100644 index 000000000..8a2099d61 --- /dev/null +++ b/packages/astro/src/@types/transformer.ts @@ -0,0 +1,23 @@ +import type { TemplateNode } from 'astro-parser'; +import type { CompileOptions } from './compiler'; + +export type VisitorFn<T = TemplateNode> = (this: { skip: () => void; remove: () => void; replace: (node: T) => void }, node: T, parent: T, type: string, index: number) => void; + +export interface NodeVisitor { + enter?: VisitorFn; + leave?: VisitorFn; +} + +export interface Transformer { + visitors?: { + html?: Record<string, NodeVisitor>; + css?: Record<string, NodeVisitor>; + }; + finalize: () => Promise<void>; +} + +export interface TransformOptions { + compileOptions: CompileOptions; + filename: string; + fileID: string; +} diff --git a/packages/astro/src/ast.ts b/packages/astro/src/ast.ts new file mode 100644 index 000000000..4f6848c89 --- /dev/null +++ b/packages/astro/src/ast.ts @@ -0,0 +1,28 @@ +import type { Attribute } from 'astro-parser'; + +// AST utility functions + +/** Get TemplateNode attribute from name */ +export function getAttr(attributes: Attribute[], name: string): Attribute | undefined { + const attr = attributes.find((a) => a.name === name); + return attr; +} + +/** Get TemplateNode attribute by value */ +export function getAttrValue(attributes: Attribute[], name: string): string | undefined { + const attr = getAttr(attributes, name); + if (attr) { + return attr.value[0]?.data; + } +} + +/** Set TemplateNode attribute value */ +export function setAttrValue(attributes: Attribute[], name: string, value: string): void { + const attr = attributes.find((a) => a.name === name); + if (attr) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + attr.value[0]!.data = value; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + attr.value[0]!.raw = value; + } +} diff --git a/packages/astro/src/build.ts b/packages/astro/src/build.ts new file mode 100644 index 000000000..392b0a920 --- /dev/null +++ b/packages/astro/src/build.ts @@ -0,0 +1,303 @@ +import type { AstroConfig, RuntimeMode } from './@types/astro'; +import type { LogOptions } from './logger'; +import type { AstroRuntime, LoadResult } from './runtime'; + +import { existsSync, promises as fsPromises } from 'fs'; +import { bold, green, yellow, underline } from 'kleur/colors'; +import path from 'path'; +import cheerio from 'cheerio'; +import { fileURLToPath } from 'url'; +import { fdir } from 'fdir'; +import { defaultLogDestination, error, info, trapWarn } from './logger.js'; +import { createRuntime } from './runtime.js'; +import { bundle, collectDynamicImports } from './build/bundle.js'; +import { generateRSS } from './build/rss.js'; +import { generateSitemap } from './build/sitemap.js'; +import { collectStatics } from './build/static.js'; +import { canonicalURL } from './build/util.js'; + + +const { mkdir, readFile, writeFile } = fsPromises; + +interface PageBuildOptions { + astroRoot: URL; + dist: URL; + filepath: URL; + runtime: AstroRuntime; + site?: string; + sitemap: boolean; + statics: Set<string>; +} + +interface PageResult { + canonicalURLs: string[]; + rss?: string; + statusCode: number; +} + +const logging: LogOptions = { + level: 'debug', + dest: defaultLogDestination, +}; + +/** Return contents of src/pages */ +async function allPages(root: URL) { + const api = new fdir() + .filter((p) => /\.(astro|md)$/.test(p)) + .withFullPaths() + .crawl(fileURLToPath(root)); + const files = await api.withPromise(); + return files as string[]; +} + +/** Utility for merging two Set()s */ +function mergeSet(a: Set<string>, b: Set<string>) { + for (let str of b) { + a.add(str); + } + return a; +} + +/** Utility for writing to file (async) */ +async function writeFilep(outPath: URL, bytes: string | Buffer, encoding: 'utf8' | null) { + const outFolder = new URL('./', outPath); + await mkdir(outFolder, { recursive: true }); + await writeFile(outPath, bytes, encoding || 'binary'); +} + +/** Utility for writing a build result to disk */ +async function writeResult(result: LoadResult, outPath: URL, encoding: null | 'utf8') { + if (result.statusCode === 500 || result.statusCode === 404) { + error(logging, 'build', result.error || result.statusCode); + } else if (result.statusCode !== 200) { + error(logging, 'build', `Unexpected load result (${result.statusCode}) for ${fileURLToPath(outPath)}`); + } else { + const bytes = result.contents; + await writeFilep(outPath, bytes, encoding); + } +} + +/** Collection utility */ +function getPageType(filepath: URL): 'collection' | 'static' { + if (/\$[^.]+.astro$/.test(filepath.pathname)) return 'collection'; + return 'static'; +} + +/** Build collection */ +async function buildCollectionPage({ astroRoot, dist, filepath, runtime, site, statics }: PageBuildOptions): Promise<PageResult> { + const rel = path.relative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro + const pagePath = `/${rel.replace(/\$([^.]+)\.astro$/, '$1')}`; + const builtURLs = new Set<string>(); // !important: internal cache that prevents building the same URLs + + /** Recursively build collection URLs */ + async function loadCollection(url: string): Promise<LoadResult | undefined> { + if (builtURLs.has(url)) return; // this stops us from recursively building the same pages over and over + const result = await runtime.load(url); + builtURLs.add(url); + if (result.statusCode === 200) { + const outPath = new URL('./' + url + '/index.html', dist); + await writeResult(result, outPath, 'utf8'); + mergeSet(statics, collectStatics(result.contents.toString('utf8'))); + } + return result; + } + + const result = (await loadCollection(pagePath)) as LoadResult; + + if (result.statusCode >= 500) { + throw new Error((result as any).error); + } + if (result.statusCode === 200 && !result.collectionInfo) { + throw new Error(`[${rel}]: Collection page must export createCollection() function`); + } + + let rss: string | undefined; + + // note: for pages that require params (/tag/:tag), we will get a 404 but will still get back collectionInfo that tell us what the URLs should be + if (result.collectionInfo) { + // build subsequent pages + await Promise.all( + [...result.collectionInfo.additionalURLs].map(async (url) => { + // for the top set of additional URLs, we render every new URL generated + const addlResult = await loadCollection(url); + builtURLs.add(url); + if (addlResult && addlResult.collectionInfo) { + // believe it or not, we may still have a few unbuilt pages left. this is our last crawl: + await Promise.all([...addlResult.collectionInfo.additionalURLs].map(async (url2) => loadCollection(url2))); + } + }) + ); + + if (result.collectionInfo.rss) { + if (!site) throw new Error(`[${rel}] createCollection() tried to generate RSS but "buildOptions.site" missing in astro.config.mjs`); + rss = generateRSS({ ...(result.collectionInfo.rss as any), site }, rel.replace(/\$([^.]+)\.astro$/, '$1')); + } + } + + return { + canonicalURLs: [...builtURLs].filter((url) => !url.endsWith('/1')), // note: canonical URLs are controlled by the collection, so these are canonical (but exclude "/1" pages as those are duplicates of the index) + statusCode: result.statusCode, + rss, + }; +} + +/** Build static page */ +async function buildStaticPage({ astroRoot, dist, filepath, runtime, sitemap, statics }: PageBuildOptions): Promise<PageResult> { + const rel = path.relative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro + const pagePath = `/${rel.replace(/\.(astro|md)$/, '')}`; + let canonicalURLs: string[] = []; + + let relPath = './' + rel.replace(/\.(astro|md)$/, '.html'); + if (!relPath.endsWith('index.html')) { + relPath = relPath.replace(/\.html$/, '/index.html'); + } + + const outPath = new URL(relPath, dist); + const result = await runtime.load(pagePath); + + await writeResult(result, outPath, 'utf8'); + + if (result.statusCode === 200) { + mergeSet(statics, collectStatics(result.contents.toString('utf8'))); + + // get Canonical URL (if user has specified one manually, use that) + if (sitemap) { + const $ = cheerio.load(result.contents); + const canonicalTag = $('link[rel="canonical"]'); + canonicalURLs.push(canonicalTag.attr('href') || pagePath.replace(/index$/, '')); + } + } + + return { + canonicalURLs, + statusCode: result.statusCode, + }; +} + +/** The primary build action */ +export async function build(astroConfig: AstroConfig): Promise<0 | 1> { + const { projectRoot, astroRoot } = astroConfig; + const pageRoot = new URL('./pages/', astroRoot); + const componentRoot = new URL('./components/', astroRoot); + const dist = new URL(astroConfig.dist + '/', projectRoot); + + const runtimeLogging: LogOptions = { + level: 'error', + dest: defaultLogDestination, + }; + + const mode: RuntimeMode = 'production'; + const runtime = await createRuntime(astroConfig, { mode, logging: runtimeLogging }); + const { runtimeConfig } = runtime; + const { backendSnowpack: snowpack } = runtimeConfig; + const resolvePackageUrl = (pkgName: string) => snowpack.getUrlForPackage(pkgName); + + const imports = new Set<string>(); + const statics = new Set<string>(); + const collectImportsOptions = { astroConfig, logging, resolvePackageUrl, mode }; + + const pages = await allPages(pageRoot); + let builtURLs: string[] = []; + + + try { + info(logging , 'build', yellow('! building pages...')); + // Vue also console.warns, this silences it. + const release = trapWarn(); + await Promise.all( + pages.map(async (pathname) => { + const filepath = new URL(`file://${pathname}`); + + const pageType = getPageType(filepath); + const pageOptions: PageBuildOptions = { astroRoot, dist, filepath, runtime, site: astroConfig.buildOptions.site, sitemap: astroConfig.buildOptions.sitemap, statics }; + if (pageType === 'collection') { + const { canonicalURLs, rss } = await buildCollectionPage(pageOptions); + builtURLs.push(...canonicalURLs); + if (rss) { + const basename = path + .relative(fileURLToPath(astroRoot) + '/pages', pathname) + .replace(/^\$/, '') + .replace(/\.astro$/, ''); + await writeFilep(new URL(`file://${path.join(fileURLToPath(dist), 'feed', basename + '.xml')}`), rss, 'utf8'); + } + } else { + const { canonicalURLs } = await buildStaticPage(pageOptions); + builtURLs.push(...canonicalURLs); + } + + mergeSet(imports, await collectDynamicImports(filepath, collectImportsOptions)); + }) + ); + info(logging, 'build', green('✔'), 'pages built.'); + release(); + } catch (err) { + error(logging, 'generate', err); + await runtime.shutdown(); + return 1; + } + + info(logging, 'build', yellow('! scanning pages...')); + for (const pathname of await allPages(componentRoot)) { + mergeSet(imports, await collectDynamicImports(new URL(`file://${pathname}`), collectImportsOptions)); + } + info(logging, 'build', green('✔'), 'pages scanned.'); + + if (imports.size > 0) { + try { + info(logging, 'build', yellow('! bundling client-side code.')); + await bundle(imports, { dist, runtime, astroConfig }); + info(logging, 'build', green('✔'), 'bundling complete.'); + } catch (err) { + error(logging, 'build', err); + await runtime.shutdown(); + return 1; + } + } + + for (let url of statics) { + const outPath = new URL('.' + url, dist); + const result = await runtime.load(url); + + await writeResult(result, outPath, null); + } + + if (existsSync(astroConfig.public)) { + info(logging, 'build', yellow(`! copying public folder...`)); + const pub = astroConfig.public; + const publicFiles = (await new fdir().withFullPaths().crawl(fileURLToPath(pub)).withPromise()) as string[]; + for (const filepath of publicFiles) { + const fileUrl = new URL(`file://${filepath}`); + const rel = path.relative(pub.pathname, fileUrl.pathname); + const outUrl = new URL('./' + rel, dist); + + const bytes = await readFile(fileUrl); + await writeFilep(outUrl, bytes, null); + } + info(logging, 'build', green('✔'), 'public folder copied.'); + } else { + if(path.basename(astroConfig.public.toString()) !=='public'){ + info(logging, 'tip', yellow(`! no public folder ${astroConfig.public} found...`)); + } + } + // build sitemap + if (astroConfig.buildOptions.sitemap && astroConfig.buildOptions.site) { + info(logging, 'build', yellow('! creating a sitemap...')); + const sitemap = generateSitemap(builtURLs.map((url) => ({ canonicalURL: canonicalURL(url, astroConfig.buildOptions.site) }))); + await writeFile(new URL('./sitemap.xml', dist), sitemap, 'utf8'); + info(logging, 'build', green('✔'), 'sitemap built.'); + } else if (astroConfig.buildOptions.sitemap) { + info(logging, 'tip', `Set "buildOptions.site" in astro.config.mjs to generate a sitemap.xml`); + } + + builtURLs.sort((a, b) => a.localeCompare(b, 'en', { numeric: true })); + info(logging, 'build', underline('Pages')); + const lastIndex = builtURLs.length - 1; + builtURLs.forEach((url, index) => { + const sep = index === 0 ? '┌' : index === lastIndex ? '└' : '├'; + info(logging, null, ' ' + sep, url === '/' ? url : url + '/'); + }); + + await runtime.shutdown(); + info(logging, 'build', bold(green('▶ Build Complete!'))); + return 0; +} diff --git a/packages/astro/src/build/bundle.ts b/packages/astro/src/build/bundle.ts new file mode 100644 index 000000000..0191e8c09 --- /dev/null +++ b/packages/astro/src/build/bundle.ts @@ -0,0 +1,313 @@ +import type { AstroConfig, RuntimeMode, ValidExtensionPlugins } from '../@types/astro'; +import type { ImportDeclaration } from '@babel/types'; +import type { InputOptions, OutputOptions } from 'rollup'; +import type { AstroRuntime } from '../runtime'; +import type { LogOptions } from '../logger'; + +import esbuild from 'esbuild'; +import { promises as fsPromises } from 'fs'; +import { fileURLToPath } from 'url'; +import { parse } from 'astro-parser'; +import { transform } from '../compiler/transform/index.js'; +import { convertMdToAstroSource } from '../compiler/index.js'; +import { getAttrValue } from '../ast.js'; +import { walk } from 'estree-walker'; +import babelParser from '@babel/parser'; +import path from 'path'; +import { rollup } from 'rollup'; +import { terser } from 'rollup-plugin-terser'; + +const { transformSync } = esbuild; +const { readFile } = fsPromises; + +type DynamicImportMap = Map<'vue' | 'react' | 'react-dom' | 'preact' | 'svelte', string>; + +/** Add framework runtimes when needed */ +async function acquireDynamicComponentImports(plugins: Set<ValidExtensionPlugins>, resolvePackageUrl: (s: string) => Promise<string>): Promise<DynamicImportMap> { + const importMap: DynamicImportMap = new Map(); + for (let plugin of plugins) { + switch (plugin) { + case 'svelte': { + importMap.set('svelte', await resolvePackageUrl('svelte')); + break; + } + case 'vue': { + importMap.set('vue', await resolvePackageUrl('vue')); + break; + } + case 'react': { + importMap.set('react', await resolvePackageUrl('react')); + importMap.set('react-dom', await resolvePackageUrl('react-dom')); + break; + } + case 'preact': { + importMap.set('preact', await resolvePackageUrl('preact')); + break; + } + } + } + return importMap; +} + +/** Evaluate mustache expression (safely) */ +function compileExpressionSafe(raw: string): string { + let { code } = transformSync(raw, { + loader: 'tsx', + jsxFactory: 'h', + jsxFragment: 'Fragment', + charset: 'utf8', + }); + return code; +} + +const defaultExtensions: Readonly<Record<string, ValidExtensionPlugins>> = { + '.jsx': 'react', + '.tsx': 'react', + '.svelte': 'svelte', + '.vue': 'vue', +}; + +interface CollectDynamic { + astroConfig: AstroConfig; + resolvePackageUrl: (s: string) => Promise<string>; + logging: LogOptions; + mode: RuntimeMode; +} + +/** Gather necessary framework runtimes for dynamic components */ +export async function collectDynamicImports(filename: URL, { astroConfig, logging, resolvePackageUrl, mode }: CollectDynamic) { + const imports = new Set<string>(); + + // Only astro files + if (!filename.pathname.endsWith('.astro') && !filename.pathname.endsWith('.md')) { + return imports; + } + + const extensions = astroConfig.extensions || defaultExtensions; + + let source = await readFile(filename, 'utf-8'); + if (filename.pathname.endsWith('.md')) { + source = await convertMdToAstroSource(source); + } + + const ast = parse(source, { + filename, + }); + + if (!ast.module) { + return imports; + } + + await transform(ast, { + filename: fileURLToPath(filename), + fileID: '', + compileOptions: { + astroConfig, + resolvePackageUrl, + logging, + mode, + }, + }); + + const componentImports: ImportDeclaration[] = []; + const components: Record<string, { plugin: ValidExtensionPlugins; type: string; specifier: string }> = {}; + const plugins = new Set<ValidExtensionPlugins>(); + + const program = babelParser.parse(ast.module.content, { + sourceType: 'module', + plugins: ['jsx', 'typescript', 'topLevelAwait'], + }).program; + + const { body } = program; + let i = body.length; + while (--i >= 0) { + const node = body[i]; + if (node.type === 'ImportDeclaration') { + componentImports.push(node); + } + } + + for (const componentImport of componentImports) { + const importUrl = componentImport.source.value; + const componentType = path.posix.extname(importUrl); + const componentName = path.posix.basename(importUrl, componentType); + const plugin = extensions[componentType] || defaultExtensions[componentType]; + plugins.add(plugin); + components[componentName] = { + plugin, + type: componentType, + specifier: importUrl, + }; + } + + const dynamic = await acquireDynamicComponentImports(plugins, resolvePackageUrl); + + /** Add dynamic component runtimes to imports */ + function appendImports(rawName: string, importUrl: URL) { + const [componentName, componentType] = rawName.split(':'); + if (!componentType) { + return; + } + + if (!components[componentName]) { + throw new Error(`Unknown Component: ${componentName}`); + } + + const defn = components[componentName]; + const fileUrl = new URL(defn.specifier, importUrl); + let rel = path.posix.relative(astroConfig.astroRoot.pathname, fileUrl.pathname); + + switch (defn.plugin) { + case 'preact': { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + imports.add(dynamic.get('preact')!); + rel = rel.replace(/\.[^.]+$/, '.js'); + break; + } + case 'react': { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + imports.add(dynamic.get('react')!); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + imports.add(dynamic.get('react-dom')!); + rel = rel.replace(/\.[^.]+$/, '.js'); + break; + } + case 'vue': { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + imports.add(dynamic.get('vue')!); + rel = rel.replace(/\.[^.]+$/, '.vue.js'); + break; + } + case 'svelte': { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + imports.add(dynamic.get('svelte')!); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + imports.add('/_astro_internal/runtime/svelte.js'); + rel = rel.replace(/\.[^.]+$/, '.svelte.js'); + break; + } + } + + imports.add(`/_astro/${rel}`); + } + + walk(ast.html, { + enter(node) { + switch (node.type) { + case 'Element': { + if (node.name !== 'script') return; + if (getAttrValue(node.attributes, 'type') !== 'module') return; + + const src = getAttrValue(node.attributes, 'src'); + + if (src && src.startsWith('/')) { + imports.add(src); + } + break; + } + + case 'MustacheTag': { + let code: string; + try { + code = compileExpressionSafe(node.content); + } catch { + return; + } + + let matches: RegExpExecArray[] = []; + let match: RegExpExecArray | null | undefined; + const H_COMPONENT_SCANNER = /h\(['"]?([A-Z].*?)['"]?,/gs; + const regex = new RegExp(H_COMPONENT_SCANNER); + while ((match = regex.exec(code))) { + matches.push(match); + } + for (const foundImport of matches.reverse()) { + const name = foundImport[1]; + appendImports(name, filename); + } + break; + } + case 'InlineComponent': { + if (/^[A-Z]/.test(node.name)) { + appendImports(node.name, filename); + return; + } + + break; + } + } + }, + }); + + return imports; +} + +interface BundleOptions { + runtime: AstroRuntime; + dist: URL; + astroConfig: AstroConfig; +} + +/** The primary bundling/optimization action */ +export async function bundle(imports: Set<string>, { runtime, dist }: BundleOptions) { + const ROOT = 'astro:root'; + const root = ` + ${[...imports].map((url) => `import '${url}';`).join('\n')} + `; + + const inputOptions: InputOptions = { + input: [...imports], + plugins: [ + { + name: 'astro:build', + resolveId(source: string, imported?: string) { + if (source === ROOT) { + return source; + } + if (source.startsWith('/')) { + return source; + } + + if (imported) { + const outUrl = new URL(source, 'http://example.com' + imported); + return outUrl.pathname; + } + + return null; + }, + async load(id: string) { + if (id === ROOT) { + return root; + } + + const result = await runtime.load(id); + + if (result.statusCode !== 200) { + return null; + } + + return result.contents.toString('utf-8'); + }, + }, + ], + }; + + const build = await rollup(inputOptions); + + const outputOptions: OutputOptions = { + dir: fileURLToPath(dist), + format: 'esm', + exports: 'named', + entryFileNames(chunk) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return chunk.facadeModuleId!.substr(1); + }, + plugins: [ + // We are using terser for the demo, but might switch to something else long term + // Look into that rather than adding options here. + terser(), + ], + }; + + await build.write(outputOptions); +} diff --git a/packages/astro/src/build/rss.ts b/packages/astro/src/build/rss.ts new file mode 100644 index 000000000..b75ed908b --- /dev/null +++ b/packages/astro/src/build/rss.ts @@ -0,0 +1,68 @@ +import type { CollectionRSS } from '../@types/astro'; +import parser from 'fast-xml-parser'; +import { canonicalURL } from './util.js'; + +/** Validates createCollection.rss */ +export function validateRSS(rss: CollectionRSS, filename: string): void { + if (!rss.title) throw new Error(`[${filename}] rss.title required`); + if (!rss.description) throw new Error(`[${filename}] rss.description required`); + if (typeof rss.item !== 'function') throw new Error(`[${filename}] rss.item() function required`); +} + +/** Generate RSS 2.0 feed */ +export function generateRSS<T>(input: { data: T[]; site: string } & CollectionRSS<T>, filename: string): string { + let xml = `<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"`; + + validateRSS(input as any, filename); + + // xmlns + if (input.xmlns) { + for (const [k, v] of Object.entries(input.xmlns)) { + xml += ` xmlns:${k}="${v}"`; + } + } + xml += `>`; + xml += `<channel>`; + + // title, description, customData + xml += `<title><![CDATA[${input.title}]]></title>`; + xml += `<description><![CDATA[${input.description}]]></description>`; + xml += `<link>${canonicalURL('/feed/' + filename + '.xml', input.site)}</link>`; + if (typeof input.customData === 'string') xml += input.customData; + + // items + if (!Array.isArray(input.data) || !input.data.length) throw new Error(`[${filename}] data() returned no items. Can’t generate RSS feed.`); + for (const item of input.data) { + xml += `<item>`; + const result = input.item(item); + // validate + if (typeof result !== 'object') throw new Error(`[${filename}] rss.item() expected to return an object, returned ${typeof result}.`); + if (!result.title) throw new Error(`[${filename}] rss.item() returned object but required "title" is missing.`); + if (!result.link) throw new Error(`[${filename}] rss.item() returned object but required "link" is missing.`); + xml += `<title><![CDATA[${result.title}]]></title>`; + xml += `<link>${canonicalURL(result.link, input.site)}</link>`; + if (result.description) xml += `<description><![CDATA[${result.description}]]></description>`; + if (result.pubDate) { + // note: this should be a Date, but if user provided a string or number, we can work with that, too. + if (typeof result.pubDate === 'number' || typeof result.pubDate === 'string') { + result.pubDate = new Date(result.pubDate); + } else if (result.pubDate instanceof Date === false) { + throw new Error('[${filename}] rss.item().pubDate must be a Date'); + } + xml += `<pubDate>${result.pubDate.toUTCString()}</pubDate>`; + } + if (typeof result.customData === 'string') xml += result.customData; + xml += `</item>`; + } + + xml += `</channel></rss>`; + + // validate user’s inputs to see if it’s valid XML + const isValid = parser.validate(xml); + if (isValid !== true) { + // If valid XML, isValid will be `true`. Otherwise, this will be an error object. Throw. + throw new Error(isValid as any); + } + + return xml; +} diff --git a/packages/astro/src/build/sitemap.ts b/packages/astro/src/build/sitemap.ts new file mode 100644 index 000000000..1cb3f3e40 --- /dev/null +++ b/packages/astro/src/build/sitemap.ts @@ -0,0 +1,15 @@ +export interface PageMeta { + /** (required) The canonical URL of the page */ + canonicalURL: string; +} + +/** Construct sitemap.xml given a set of URLs */ +export function generateSitemap(pages: PageMeta[]): string { + let sitemap = `<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`; + pages.sort((a, b) => a.canonicalURL.localeCompare(b.canonicalURL, 'en', { numeric: true })); // sort alphabetically + for (const page of pages) { + sitemap += `<url><loc>${page.canonicalURL}</loc></url>`; + } + sitemap += `</urlset>\n`; + return sitemap; +} diff --git a/packages/astro/src/build/static.ts b/packages/astro/src/build/static.ts new file mode 100644 index 000000000..af99c33cb --- /dev/null +++ b/packages/astro/src/build/static.ts @@ -0,0 +1,28 @@ +import type { Element } from 'domhandler'; +import cheerio from 'cheerio'; + +/** Given an HTML string, collect <link> and <img> tags */ +export function collectStatics(html: string) { + const statics = new Set<string>(); + + const $ = cheerio.load(html); + + const append = (el: Element, attr: string) => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const value: string = $(el).attr(attr)!; + if (value.startsWith('http') || $(el).attr('rel') === 'alternate') { + return; + } + statics.add(value); + }; + + $('link[href]').each((i, el) => { + append(el, 'href'); + }); + + $('img[src]').each((i, el) => { + append(el, 'src'); + }); + + return statics; +} diff --git a/packages/astro/src/build/util.ts b/packages/astro/src/build/util.ts new file mode 100644 index 000000000..505e6f183 --- /dev/null +++ b/packages/astro/src/build/util.ts @@ -0,0 +1,9 @@ +import path from 'path'; + +/** Normalize URL to its canonical form */ +export function canonicalURL(url: string, base?: string): string { + return new URL( + path.extname(url) ? url : url.replace(/(\/+)?$/, '/'), // add trailing slash if there’s no extension + base + ).href; +} diff --git a/packages/astro/src/cli.ts b/packages/astro/src/cli.ts new file mode 100644 index 000000000..be0dfe27a --- /dev/null +++ b/packages/astro/src/cli.ts @@ -0,0 +1,127 @@ +/* eslint-disable no-console */ +import type { AstroConfig } from './@types/astro'; + +import * as colors from 'kleur/colors'; +import { promises as fsPromises } from 'fs'; +import yargs from 'yargs-parser'; + +import { loadConfig } from './config.js'; +import { build } from './build.js'; +import devServer from './dev.js'; + +const { readFile } = fsPromises; +const buildAndExit = async (...args: Parameters<typeof build>) => { + const ret = await build(...args); + process.exit(ret); +}; + +type Arguments = yargs.Arguments; +type cliCommand = 'help' | 'version' | 'dev' | 'build'; +interface CLIState { + cmd: cliCommand; + options: { + projectRoot?: string; + sitemap?: boolean; + port?: number; + config?: string; + }; +} + +/** Determine which action the user requested */ +function resolveArgs(flags: Arguments): CLIState { + const options: CLIState['options'] = { + projectRoot: typeof flags.projectRoot === 'string' ? flags.projectRoot: undefined, + sitemap: typeof flags.sitemap === 'boolean' ? flags.sitemap : undefined, + port: typeof flags.port === 'number' ? flags.port : undefined, + config: typeof flags.config === 'string' ? flags.config : undefined + }; + + if (flags.version) { + return { cmd: 'version', options }; + } else if (flags.help) { + return { cmd: 'help', options }; + } + + const cmd = flags._[2]; + switch (cmd) { + case 'dev': + return { cmd: 'dev', options }; + case 'build': + return { cmd: 'build', options }; + default: + return { cmd: 'help', options }; + } +} + +/** Display --help flag */ +function printHelp() { + console.error(` ${colors.bold('astro')} - Futuristic web development tool. + + ${colors.bold('Commands:')} + astro dev Run Astro in development mode. + astro build Build a pre-compiled production version of your site. + + ${colors.bold('Flags:')} + --config <path> Specify the path to the Astro config file. + --project-root <path> Specify the path to the project root folder. + --no-sitemap Disable sitemap generation (build only). + --version Show the version number and exit. + --help Show this help message. +`); +} + +/** Display --version flag */ +async function printVersion() { + const pkg = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf-8')); + console.error(pkg.version); +} + +/** Merge CLI flags & config options (CLI flags take priority) */ +function mergeCLIFlags(astroConfig: AstroConfig, flags: CLIState['options']) { + if (typeof flags.sitemap === 'boolean') astroConfig.buildOptions.sitemap = flags.sitemap; + if (typeof flags.port === 'number') astroConfig.devOptions.port = flags.port; +} + +/** Handle `astro run` command */ +async function runCommand(rawRoot: string, cmd: (a: AstroConfig) => Promise<void>, options: CLIState['options']) { + try { + const projectRoot = options.projectRoot || rawRoot; + const astroConfig = await loadConfig(projectRoot, options.config); + mergeCLIFlags(astroConfig, options); + + return cmd(astroConfig); + } catch (err) { + console.error(colors.red(err.toString() || err)); + process.exit(1); + } +} + +const cmdMap = new Map([ + ['build', buildAndExit], + ['dev', devServer], +]); + +/** The primary CLI action */ +export async function cli(args: string[]) { + const flags = yargs(args); + const state = resolveArgs(flags); + + switch (state.cmd) { + case 'help': { + printHelp(); + process.exit(1); + break; + } + case 'version': { + await printVersion(); + process.exit(0); + break; + } + case 'build': + case 'dev': { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const cmd = cmdMap.get(state.cmd)!; + runCommand(flags._[3], cmd, state.options); + } + } +} diff --git a/packages/astro/src/compiler/codegen/content.ts b/packages/astro/src/compiler/codegen/content.ts new file mode 100644 index 000000000..fb8f9e307 --- /dev/null +++ b/packages/astro/src/compiler/codegen/content.ts @@ -0,0 +1,78 @@ +import path from 'path'; +import { fdir, PathsOutput } from 'fdir'; + +/** + * Handling for import.meta.glob and import.meta.globEager + */ + +interface GlobOptions { + namespace: string; + filename: string; +} + +interface GlobResult { + /** Array of import statements to inject */ + imports: Set<string>; + /** Replace original code with */ + code: string; +} + +const crawler = new fdir(); + +/** General glob handling */ +function globSearch(spec: string, { filename }: { filename: string }): string[] { + try { + // Note: fdir’s glob requires you to do some work finding the closest non-glob folder. + // For example, this fails: .glob("./post/*.md").crawl("/…/src/pages") ❌ + // …but this doesn’t: .glob("*.md").crawl("/…/src/pages/post") ✅ + let globDir = ''; + let glob = spec; + for (const part of spec.split('/')) { + if (!part.includes('*')) { + // iterate through spec until first '*' is reached + globDir = path.posix.join(globDir, part); // this must be POSIX-style + glob = glob.replace(`${part}/`, ''); // move parent dirs off spec, and onto globDir + } else { + // at first '*', exit + break; + } + } + + const cwd = path.join(path.dirname(filename), globDir.replace(/\//g, path.sep)); // this must match OS (could be '/' or '\') + let found = crawler.glob(glob).crawl(cwd).sync() as PathsOutput; + if (!found.length) { + throw new Error(`No files matched "${spec}" from ${filename}`); + } + return found.map((importPath) => { + if (importPath.startsWith('http') || importPath.startsWith('.')) return importPath; + return `./` + globDir + '/' + importPath; + }); + } catch (err) { + throw new Error(`No files matched "${spec}" from ${filename}`); + } +} + +/** Astro.fetchContent() */ +export function fetchContent(spec: string, { namespace, filename }: GlobOptions): GlobResult { + let code = ''; + const imports = new Set<string>(); + const importPaths = globSearch(spec, { filename }); + + // gather imports + importPaths.forEach((importPath, j) => { + const id = `${namespace}_${j}`; + imports.add(`import { __content as ${id} } from '${importPath}';`); + + // add URL if this appears within the /pages/ directory (probably can be improved) + const fullPath = path.resolve(path.dirname(filename), importPath); + if (fullPath.includes(`${path.sep}pages${path.sep}`)) { + const url = importPath.replace(/^\./, '').replace(/\.md$/, ''); + imports.add(`${id}.url = '${url}';`); + } + }); + + // generate replacement code + code += `${namespace} = [${importPaths.map((_, j) => `${namespace}_${j}`).join(',')}];\n`; + + return { imports, code }; +} diff --git a/packages/astro/src/compiler/codegen/index.ts b/packages/astro/src/compiler/codegen/index.ts new file mode 100644 index 000000000..6caed85a3 --- /dev/null +++ b/packages/astro/src/compiler/codegen/index.ts @@ -0,0 +1,686 @@ +import type { CompileOptions } from '../../@types/compiler'; +import type { AstroConfig, ValidExtensionPlugins } from '../../@types/astro'; +import type { Ast, Script, Style, TemplateNode } from 'astro-parser'; +import type { TransformResult } from '../../@types/astro'; + +import eslexer from 'es-module-lexer'; +import esbuild from 'esbuild'; +import path from 'path'; +import { walk } from 'estree-walker'; +import _babelGenerator from '@babel/generator'; +import babelParser from '@babel/parser'; +import { codeFrameColumns } from '@babel/code-frame'; +import * as babelTraverse from '@babel/traverse'; +import { ImportDeclaration, ExportNamedDeclaration, VariableDeclarator, Identifier } from '@babel/types'; +import { warn } from '../../logger.js'; +import { fetchContent } from './content.js'; +import { isFetchContent } from './utils.js'; +import { yellow } from 'kleur/colors'; + +const traverse: typeof babelTraverse.default = (babelTraverse.default as any).default; +const babelGenerator: typeof _babelGenerator = + // @ts-ignore + _babelGenerator.default; +const { transformSync } = esbuild; + +interface Attribute { + start: number; + end: number; + type: 'Attribute'; + name: string; + value: TemplateNode[] | boolean; +} + +interface CodeGenOptions { + compileOptions: CompileOptions; + filename: string; + fileID: string; +} + +/** Format Astro internal import URL */ +function internalImport(internalPath: string) { + return `/_astro_internal/${internalPath}`; +} + +/** Retrieve attributes from TemplateNode */ +function getAttributes(attrs: Attribute[]): Record<string, string> { + let result: Record<string, string> = {}; + for (const attr of attrs) { + if (attr.value === true) { + result[attr.name] = JSON.stringify(attr.value); + continue; + } + if (attr.value === false || attr.value === undefined) { + // note: attr.value shouldn’t be `undefined`, but a bad transform would cause a compile error here, so prevent that + continue; + } + if (attr.value.length > 1) { + result[attr.name] = + '(' + + attr.value + .map((v: TemplateNode) => { + if (v.content) { + return v.content; + } else { + return JSON.stringify(getTextFromAttribute(v)); + } + }) + .join('+') + + ')'; + continue; + } + const val = attr.value[0]; + if (!val) { + result[attr.name] = '(' + val + ')'; + continue; + } + switch (val.type) { + case 'MustacheTag': { + // FIXME: this won't work when JSX element can appear in attributes (rare but possible). + result[attr.name] = '(' + val.expression.codeChunks[0] + ')'; + continue; + } + case 'Text': + result[attr.name] = JSON.stringify(getTextFromAttribute(val)); + continue; + default: + throw new Error(`UNKNOWN: ${val.type}`); + } + } + return result; +} + +/** Get value from a TemplateNode Attribute (text attributes only!) */ +function getTextFromAttribute(attr: any): string { + switch (attr.type) { + case 'Text': { + if (attr.raw !== undefined) { + return attr.raw; + } + if (attr.data !== undefined) { + return attr.data; + } + break; + } + case 'MustacheTag': { + // FIXME: this won't work when JSX element can appear in attributes (rare but possible). + return attr.expression.codeChunks[0]; + } + } + throw new Error(`Unknown attribute type ${attr.type}`); +} + +/** Convert TemplateNode attributes to string */ +function generateAttributes(attrs: Record<string, string>): string { + let result = '{'; + for (const [key, val] of Object.entries(attrs)) { + result += JSON.stringify(key) + ':' + val + ','; + } + return result + '}'; +} + +interface ComponentInfo { + type: string; + url: string; + plugin: string | undefined; +} + +const defaultExtensions: Readonly<Record<string, ValidExtensionPlugins>> = { + '.astro': 'astro', + '.jsx': 'react', + '.tsx': 'react', + '.vue': 'vue', + '.svelte': 'svelte', +}; + +type DynamicImportMap = Map<'vue' | 'react' | 'react-dom' | 'preact' | 'svelte', string>; + +interface GetComponentWrapperOptions { + filename: string; + astroConfig: AstroConfig; + dynamicImports: DynamicImportMap; +} + +/** Generate Astro-friendly component import */ +function getComponentWrapper(_name: string, { type, plugin, url }: ComponentInfo, opts: GetComponentWrapperOptions) { + const { astroConfig, dynamicImports, filename } = opts; + const { astroRoot } = astroConfig; + const [name, kind] = _name.split(':'); + const currFileUrl = new URL(`file://${filename}`); + + if (!plugin) { + throw new Error(`No supported plugin found for ${type ? `extension ${type}` : `${url} (try adding an extension)`}`); + } + + const getComponentUrl = (ext = '.js') => { + const outUrl = new URL(url, currFileUrl); + return '/_astro/' + path.posix.relative(astroRoot.pathname, outUrl.pathname).replace(/\.[^.]+$/, ext); + }; + + switch (plugin) { + case 'astro': { + if (kind) { + throw new Error(`Astro does not support :${kind}`); + } + return { + wrapper: name, + wrapperImport: ``, + }; + } + case 'preact': { + if (['load', 'idle', 'visible'].includes(kind)) { + return { + wrapper: `__preact_${kind}(${name}, ${JSON.stringify({ + componentUrl: getComponentUrl(), + componentExport: 'default', + frameworkUrls: { + preact: dynamicImports.get('preact'), + }, + })})`, + wrapperImport: `import {__preact_${kind}} from '${internalImport('render/preact.js')}';`, + }; + } + + return { + wrapper: `__preact_static(${name})`, + wrapperImport: `import {__preact_static} from '${internalImport('render/preact.js')}';`, + }; + } + case 'react': { + if (['load', 'idle', 'visible'].includes(kind)) { + return { + wrapper: `__react_${kind}(${name}, ${JSON.stringify({ + componentUrl: getComponentUrl(), + componentExport: 'default', + frameworkUrls: { + react: dynamicImports.get('react'), + 'react-dom': dynamicImports.get('react-dom'), + }, + })})`, + wrapperImport: `import {__react_${kind}} from '${internalImport('render/react.js')}';`, + }; + } + + return { + wrapper: `__react_static(${name})`, + wrapperImport: `import {__react_static} from '${internalImport('render/react.js')}';`, + }; + } + case 'svelte': { + if (['load', 'idle', 'visible'].includes(kind)) { + return { + wrapper: `__svelte_${kind}(${name}, ${JSON.stringify({ + componentUrl: getComponentUrl('.svelte.js'), + componentExport: 'default', + frameworkUrls: { + 'svelte-runtime': internalImport('runtime/svelte.js'), + }, + })})`, + wrapperImport: `import {__svelte_${kind}} from '${internalImport('render/svelte.js')}';`, + }; + } + + return { + wrapper: `__svelte_static(${name})`, + wrapperImport: `import {__svelte_static} from '${internalImport('render/svelte.js')}';`, + }; + } + case 'vue': { + if (['load', 'idle', 'visible'].includes(kind)) { + return { + wrapper: `__vue_${kind}(${name}, ${JSON.stringify({ + componentUrl: getComponentUrl('.vue.js'), + componentExport: 'default', + frameworkUrls: { + vue: dynamicImports.get('vue'), + }, + })})`, + wrapperImport: `import {__vue_${kind}} from '${internalImport('render/vue.js')}';`, + }; + } + + return { + wrapper: `__vue_static(${name})`, + wrapperImport: `import {__vue_static} from '${internalImport('render/vue.js')}';`, + }; + } + default: { + throw new Error(`Unknown component type`); + } + } +} + +/** Evaluate expression (safely) */ +function compileExpressionSafe(raw: string): string { + let { code } = transformSync(raw, { + loader: 'tsx', + jsxFactory: 'h', + jsxFragment: 'Fragment', + charset: 'utf8', + }); + return code; +} + +/** Build dependency map of dynamic component runtime frameworks */ +async function acquireDynamicComponentImports(plugins: Set<ValidExtensionPlugins>, resolvePackageUrl: (s: string) => Promise<string>): Promise<DynamicImportMap> { + const importMap: DynamicImportMap = new Map(); + for (let plugin of plugins) { + switch (plugin) { + case 'vue': { + importMap.set('vue', await resolvePackageUrl('vue')); + break; + } + case 'react': { + importMap.set('react', await resolvePackageUrl('react')); + importMap.set('react-dom', await resolvePackageUrl('react-dom')); + break; + } + case 'preact': { + importMap.set('preact', await resolvePackageUrl('preact')); + break; + } + case 'svelte': { + importMap.set('svelte', await resolvePackageUrl('svelte')); + break; + } + } + } + return importMap; +} + +type Components = Record<string, { type: string; url: string; plugin: string | undefined }>; + +interface CompileResult { + script: string; + componentPlugins: Set<ValidExtensionPlugins>; + createCollection?: string; +} + +interface CodegenState { + filename: string; + components: Components; + css: string[]; + importExportStatements: Set<string>; + dynamicImports: DynamicImportMap; +} + +/** Compile/prepare Astro frontmatter scripts */ +function compileModule(module: Script, state: CodegenState, compileOptions: CompileOptions): CompileResult { + const { extensions = defaultExtensions } = compileOptions; + + const componentImports: ImportDeclaration[] = []; + const componentProps: VariableDeclarator[] = []; + const componentExports: ExportNamedDeclaration[] = []; + + const contentImports = new Map<string, { spec: string; declarator: string }>(); + + let script = ''; + let propsStatement = ''; + let contentCode = ''; // code for handling Astro.fetchContent(), if any; + let createCollection = ''; // function for executing collection + const componentPlugins = new Set<ValidExtensionPlugins>(); + + if (module) { + const parseOptions: babelParser.ParserOptions = { + sourceType: 'module', + plugins: ['jsx', 'typescript', 'topLevelAwait'], + }; + let parseResult; + try { + parseResult = babelParser.parse(module.content, parseOptions); + } catch (err) { + const location = { start: err.loc }; + const frame = codeFrameColumns(module.content, location); + err.frame = frame; + err.filename = state.filename; + err.start = err.loc; + throw err; + } + const program = parseResult.program; + + const { body } = program; + let i = body.length; + while (--i >= 0) { + const node = body[i]; + switch (node.type) { + case 'ExportNamedDeclaration': { + if (!node.declaration) break; + // const replacement = extract_exports(node); + + if (node.declaration.type === 'VariableDeclaration') { + // case 1: prop (export let title) + + const declaration = node.declaration.declarations[0]; + if ((declaration.id as Identifier).name === '__layout' || (declaration.id as Identifier).name === '__content') { + componentExports.push(node); + } else { + componentProps.push(declaration); + } + body.splice(i, 1); + } else if (node.declaration.type === 'FunctionDeclaration') { + // case 2: createCollection (export async function) + if (!node.declaration.id || node.declaration.id.name !== 'createCollection') break; + createCollection = module.content.substring(node.declaration.start || 0, node.declaration.end || 0); + + // remove node + body.splice(i, 1); + } + break; + } + case 'FunctionDeclaration': { + break; + } + case 'ImportDeclaration': { + componentImports.push(node); + body.splice(i, 1); // remove node + break; + } + case 'VariableDeclaration': { + for (const declaration of node.declarations) { + // only select Astro.fetchContent() calls here. this utility filters those out for us. + if (!isFetchContent(declaration)) continue; + + // remove node + body.splice(i, 1); + + // a bit of munging + let { id, init } = declaration; + if (!id || !init || id.type !== 'Identifier') continue; + if (init.type === 'AwaitExpression') { + init = init.argument; + const shortname = path.relative(compileOptions.astroConfig.projectRoot.pathname, state.filename); + warn(compileOptions.logging, shortname, yellow('awaiting Astro.fetchContent() not necessary')); + } + if (init.type !== 'CallExpression') continue; + + // gather data + const namespace = id.name; + + if ((init as any).arguments[0].type !== 'StringLiteral') { + throw new Error(`[Astro.fetchContent] Only string literals allowed, ex: \`Astro.fetchContent('./post/*.md')\`\n ${state.filename}`); + } + const spec = (init as any).arguments[0].value; + if (typeof spec === 'string') contentImports.set(namespace, { spec, declarator: node.kind }); + } + break; + } + } + } + + for (const componentImport of componentImports) { + const importUrl = componentImport.source.value; + const componentType = path.posix.extname(importUrl); + const specifier = componentImport.specifiers[0]; + if (!specifier) continue; // this is unused + // set componentName to default import if used (user), or use filename if no default import (mostly internal use) + const componentName = specifier.type === 'ImportDefaultSpecifier' ? specifier.local.name : path.posix.basename(importUrl, componentType); + const plugin = extensions[componentType] || defaultExtensions[componentType]; + state.components[componentName] = { + type: componentType, + plugin, + url: importUrl, + }; + if (plugin) { + componentPlugins.add(plugin); + } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + state.importExportStatements.add(module.content.slice(componentImport.start!, componentImport.end!)); + } + for (const componentImport of componentExports) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + state.importExportStatements.add(module.content.slice(componentImport.start!, componentImport.end!)); + } + + if (componentProps.length > 0) { + propsStatement = 'let {'; + for (const componentExport of componentProps) { + propsStatement += `${(componentExport.id as Identifier).name}`; + if (componentExport.init) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + propsStatement += `= ${babelGenerator(componentExport.init!).code}`; + } + propsStatement += `,`; + } + propsStatement += `} = props;\n`; + } + + // handle createCollection, if any + if (createCollection) { + // TODO: improve this? while transforming in-place isn’t great, this happens at most once per-route + const ast = babelParser.parse(createCollection, { + sourceType: 'module', + }); + traverse(ast, { + enter({ node }) { + switch (node.type) { + case 'VariableDeclaration': { + for (const declaration of node.declarations) { + // only select Astro.fetchContent() calls here. this utility filters those out for us. + if (!isFetchContent(declaration)) continue; + + // a bit of munging + let { id, init } = declaration; + if (!id || !init || id.type !== 'Identifier') continue; + if (init.type === 'AwaitExpression') { + init = init.argument; + const shortname = path.relative(compileOptions.astroConfig.projectRoot.pathname, state.filename); + warn(compileOptions.logging, shortname, yellow('awaiting Astro.fetchContent() not necessary')); + } + if (init.type !== 'CallExpression') continue; + + // gather data + const namespace = id.name; + + if ((init as any).arguments[0].type !== 'StringLiteral') { + throw new Error(`[Astro.fetchContent] Only string literals allowed, ex: \`Astro.fetchContent('./post/*.md')\`\n ${state.filename}`); + } + const spec = (init as any).arguments[0].value; + if (typeof spec !== 'string') break; + + const globResult = fetchContent(spec, { namespace, filename: state.filename }); + + let imports = ''; + for (const importStatement of globResult.imports) { + imports += importStatement + '\n'; + } + + createCollection = + imports + '\nexport ' + createCollection.substring(0, declaration.start || 0) + globResult.code + createCollection.substring(declaration.end || 0); + } + break; + } + } + }, + }); + } + + // Astro.fetchContent() + for (const [namespace, { spec }] of contentImports.entries()) { + const globResult = fetchContent(spec, { namespace, filename: state.filename }); + for (const importStatement of globResult.imports) { + state.importExportStatements.add(importStatement); + } + contentCode += globResult.code; + } + + script = propsStatement + contentCode + babelGenerator(program).code; + } + + return { + script, + componentPlugins, + createCollection: createCollection || undefined, + }; +} + +/** Compile styles */ +function compileCss(style: Style, state: CodegenState) { + walk(style, { + enter(node: TemplateNode) { + if (node.type === 'Style') { + state.css.push(node.content.styles); // if multiple <style> tags, combine together + this.skip(); + } + }, + leave(node: TemplateNode) { + if (node.type === 'Style') { + this.remove(); // this will be optimized in a global CSS file; remove so it‘s not accidentally inlined + } + }, + }); +} + +/** Compile page markup */ +function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOptions: CompileOptions) { + const { components, css, importExportStatements, dynamicImports, filename } = state; + const { astroConfig } = compileOptions; + + let outSource = ''; + walk(enterNode, { + enter(node: TemplateNode) { + switch (node.type) { + case 'Expression': { + let children: string[] = []; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + for (const child of node.children!) { + children.push(compileHtml(child, state, compileOptions)); + } + let raw = ''; + let nextChildIndex = 0; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + for (const chunk of node.codeChunks!) { + raw += chunk; + if (nextChildIndex < children.length) { + raw += children[nextChildIndex++]; + } + } + // TODO Do we need to compile this now, or should we compile the entire module at the end? + let code = compileExpressionSafe(raw).trim().replace(/\;$/, ''); + outSource += `,(${code})`; + this.skip(); + break; + } + case 'MustacheTag': + case 'Comment': + return; + case 'Fragment': + break; + case 'Slot': + case 'Head': + case 'InlineComponent': + case 'Title': + case 'Element': { + const name: string = node.name; + if (!name) { + throw new Error('AHHHH'); + } + const attributes = getAttributes(node.attributes); + + outSource += outSource === '' ? '' : ','; + if (node.type === 'Slot') { + outSource += `(children`; + return; + } + const COMPONENT_NAME_SCANNER = /^[A-Z]/; + if (!COMPONENT_NAME_SCANNER.test(name)) { + outSource += `h("${name}", ${attributes ? generateAttributes(attributes) : 'null'}`; + return; + } + const [componentName, componentKind] = name.split(':'); + const componentImportData = components[componentName]; + if (!componentImportData) { + throw new Error(`Unknown Component: ${componentName}`); + } + const { wrapper, wrapperImport } = getComponentWrapper(name, components[componentName], { astroConfig, dynamicImports, filename }); + if (wrapperImport) { + importExportStatements.add(wrapperImport); + } + + outSource += `h(${wrapper}, ${attributes ? generateAttributes(attributes) : 'null'}`; + return; + } + case 'Attribute': { + this.skip(); + return; + } + case 'Style': { + css.push(node.content.styles); // if multiple <style> tags, combine together + this.skip(); + return; + } + case 'Text': { + const text = getTextFromAttribute(node); + if (!text.trim()) { + return; + } + outSource += ',' + JSON.stringify(text); + return; + } + default: + throw new Error('Unexpected (enter) node type: ' + node.type); + } + }, + leave(node, parent, prop, index) { + switch (node.type) { + case 'Text': + case 'Attribute': + case 'Comment': + case 'Fragment': + case 'Expression': + case 'MustacheTag': + return; + case 'Slot': + case 'Head': + case 'Body': + case 'Title': + case 'Element': + case 'InlineComponent': + outSource += ')'; + return; + case 'Style': { + this.remove(); // this will be optimized in a global CSS file; remove so it‘s not accidentally inlined + return; + } + default: + throw new Error('Unexpected (leave) node type: ' + node.type); + } + }, + }); + + return outSource; +} + +/** + * Codegen + * Step 3/3 in Astro SSR. + * This is the final pass over a document AST before it‘s converted to an h() function + * and handed off to Snowpack to build. + * @param {Ast} AST The parsed AST to crawl + * @param {object} CodeGenOptions + */ +export async function codegen(ast: Ast, { compileOptions, filename }: CodeGenOptions): Promise<TransformResult> { + await eslexer.init; + + const state: CodegenState = { + filename, + components: {}, + css: [], + importExportStatements: new Set(), + dynamicImports: new Map(), + }; + + const { script, componentPlugins, createCollection } = compileModule(ast.module, state, compileOptions); + state.dynamicImports = await acquireDynamicComponentImports(componentPlugins, compileOptions.resolvePackageUrl); + + compileCss(ast.css, state); + + const html = compileHtml(ast.html, state, compileOptions); + + return { + script: script, + imports: Array.from(state.importExportStatements), + html, + css: state.css.length ? state.css.join('\n\n') : undefined, + createCollection, + }; +} diff --git a/packages/astro/src/compiler/codegen/utils.ts b/packages/astro/src/compiler/codegen/utils.ts new file mode 100644 index 000000000..e1c558bc4 --- /dev/null +++ b/packages/astro/src/compiler/codegen/utils.ts @@ -0,0 +1,39 @@ +/** + * Codegen utils + */ + +import type { VariableDeclarator } from '@babel/types'; + +/** Is this an import.meta.* built-in? You can pass an optional 2nd param to see if the name matches as well. */ +export function isImportMetaDeclaration(declaration: VariableDeclarator, metaName?: string): boolean { + let { init } = declaration; + if (!init) return false; // definitely not import.meta + // this could be `await import.meta`; if so, evaluate that: + if (init.type === 'AwaitExpression') { + init = init.argument; + } + // continue evaluating + if (init.type !== 'CallExpression' || init.callee.type !== 'MemberExpression' || init.callee.object.type !== 'MetaProperty') return false; + // optional: if metaName specified, match that + if (metaName && (init.callee.property.type !== 'Identifier' || init.callee.property.name !== metaName)) return false; + return true; +} + +/** Is this an Astro.fetchContent() call? */ +export function isFetchContent(declaration: VariableDeclarator): boolean { + let { init } = declaration; + if (!init) return false; // definitely not import.meta + // this could be `await import.meta`; if so, evaluate that: + if (init.type === 'AwaitExpression') { + init = init.argument; + } + // continue evaluating + if ( + init.type !== 'CallExpression' || + init.callee.type !== 'MemberExpression' || + (init.callee.object as any).name !== 'Astro' || + (init.callee.property as any).name !== 'fetchContent' + ) + return false; + return true; +} diff --git a/packages/astro/src/compiler/index.ts b/packages/astro/src/compiler/index.ts new file mode 100644 index 000000000..bb8ac61d4 --- /dev/null +++ b/packages/astro/src/compiler/index.ts @@ -0,0 +1,176 @@ +import type { CompileResult, TransformResult } from '../@types/astro'; +import type { CompileOptions } from '../@types/compiler.js'; + +import path from 'path'; +import micromark from 'micromark'; +import gfmSyntax from 'micromark-extension-gfm'; +import matter from 'gray-matter'; +import gfmHtml from 'micromark-extension-gfm/html.js'; + +import { parse } from 'astro-parser'; +import { createMarkdownHeadersCollector } from './markdown/micromark-collect-headers.js'; +import { encodeMarkdown } from './markdown/micromark-encode.js'; +import { encodeAstroMdx } from './markdown/micromark-mdx-astro.js'; +import { transform } from './transform/index.js'; +import { codegen } from './codegen/index.js'; + +/** Return Astro internal import URL */ +function internalImport(internalPath: string) { + return `/_astro_internal/${internalPath}`; +} + +interface ConvertAstroOptions { + compileOptions: CompileOptions; + filename: string; + fileID: string; +} + +/** + * .astro -> .jsx + * Core function processing .astro files. Initiates all 3 phases of compilation: + * 1. Parse + * 2. Transform + * 3. Codegen + */ +async function convertAstroToJsx(template: string, opts: ConvertAstroOptions): Promise<TransformResult> { + const { filename } = opts; + + // 1. Parse + const ast = parse(template, { + filename, + }); + + // 2. Transform the AST + await transform(ast, opts); + + // 3. Turn AST into JSX + return await codegen(ast, opts); +} + +/** + * .md -> .astro source + */ +export async function convertMdToAstroSource(contents: string): Promise<string> { + const { data: frontmatterData, content } = matter(contents); + const { headers, headersExtension } = createMarkdownHeadersCollector(); + const { htmlAstro, mdAstro } = encodeAstroMdx(); + const mdHtml = micromark(content, { + allowDangerousHtml: true, + extensions: [gfmSyntax(), ...htmlAstro], + htmlExtensions: [gfmHtml, encodeMarkdown, headersExtension, mdAstro], + }); + + // TODO: Warn if reserved word is used in "frontmatterData" + const contentData: any = { + ...frontmatterData, + headers, + source: content, + }; + + let imports = ''; + for (let [ComponentName, specifier] of Object.entries(frontmatterData.import || {})) { + imports += `import ${ComponentName} from '${specifier}';\n`; + } + + // </script> can't be anywhere inside of a JS string, otherwise the HTML parser fails. + // Break it up here so that the HTML parser won't detect it. + const stringifiedSetupContext = JSON.stringify(contentData).replace(/\<\/script\>/g, `</scrip" + "t>`); + + return `--- + ${imports} + ${frontmatterData.layout ? `import {__renderPage as __layout} from '${frontmatterData.layout}';` : 'const __layout = undefined;'} + export const __content = ${stringifiedSetupContext}; +--- +<section>${mdHtml}</section>`; +} + +/** + * .md -> .jsx + * Core function processing Markdown, but along the way also calls convertAstroToJsx(). + */ +async function convertMdToJsx( + contents: string, + { compileOptions, filename, fileID }: { compileOptions: CompileOptions; filename: string; fileID: string } +): Promise<TransformResult> { + const raw = await convertMdToAstroSource(contents); + const convertOptions = { compileOptions, filename, fileID }; + return await convertAstroToJsx(raw, convertOptions); +} + +type SupportedExtensions = '.astro' | '.md'; + +/** Given a file, process it either as .astro or .md. */ +async function transformFromSource( + contents: string, + { compileOptions, filename, projectRoot }: { compileOptions: CompileOptions; filename: string; projectRoot: string } +): Promise<TransformResult> { + const fileID = path.relative(projectRoot, filename); + switch (path.extname(filename) as SupportedExtensions) { + case '.astro': + return await convertAstroToJsx(contents, { compileOptions, filename, fileID }); + case '.md': + return await convertMdToJsx(contents, { compileOptions, filename, fileID }); + default: + throw new Error('Not Supported!'); + } +} + +/** Return internal code that gets processed in Snowpack */ +export async function compileComponent( + source: string, + { compileOptions, filename, projectRoot }: { compileOptions: CompileOptions; filename: string; projectRoot: string } +): Promise<CompileResult> { + const result = await transformFromSource(source, { compileOptions, filename, projectRoot }); + + // return template + let modJsx = ` +import fetch from 'node-fetch'; + +// <script astro></script> +${result.imports.join('\n')} + +// \`__render()\`: Render the contents of the Astro module. +import { h, Fragment } from '${internalImport('h.js')}'; +const __astroRequestSymbol = Symbol('astro.request'); +async function __render(props, ...children) { + const Astro = { + request: props[__astroRequestSymbol] + }; + + ${result.script} + return h(Fragment, null, ${result.html}); +} +export default __render; + +${result.createCollection || ''} + +// \`__renderPage()\`: Render the contents of the Astro module as a page. This is a special flow, +// triggered by loading a component directly by URL. +export async function __renderPage({request, children, props}) { + const currentChild = { + layout: typeof __layout === 'undefined' ? undefined : __layout, + content: typeof __content === 'undefined' ? undefined : __content, + __render, + }; + + props[__astroRequestSymbol] = request; + const childBodyResult = await currentChild.__render(props, children); + + // find layout, if one was given. + if (currentChild.layout) { + return currentChild.layout({ + request, + props: {content: currentChild.content}, + children: [childBodyResult], + }); + } + + return childBodyResult; +};\n`; + + return { + result, + contents: modJsx, + css: result.css, + }; +} diff --git a/packages/astro/src/compiler/markdown/micromark-collect-headers.ts b/packages/astro/src/compiler/markdown/micromark-collect-headers.ts new file mode 100644 index 000000000..69781231a --- /dev/null +++ b/packages/astro/src/compiler/markdown/micromark-collect-headers.ts @@ -0,0 +1,38 @@ +import slugger from 'github-slugger'; + +/** + * Create Markdown Headers Collector + * NOTE: micromark has terrible TS types. Instead of fighting with the + * limited/broken TS types that they ship, we just reach for our good friend, "any". + */ +export function createMarkdownHeadersCollector() { + const headers: any[] = []; + let currentHeader: any; + return { + headers, + headersExtension: { + enter: { + atxHeading(node: any) { + currentHeader = {}; + headers.push(currentHeader); + this.buffer(); + }, + atxHeadingSequence(node: any) { + currentHeader.depth = this.sliceSerialize(node).length; + }, + atxHeadingText(node: any) { + currentHeader.text = this.sliceSerialize(node); + }, + } as any, + exit: { + atxHeading(node: any) { + currentHeader.slug = slugger.slug(currentHeader.text); + this.resume(); + this.tag(`<h${currentHeader.depth} id="${currentHeader.slug}">`); + this.raw(currentHeader.text); + this.tag(`</h${currentHeader.depth}>`); + }, + } as any, + } as any, + }; +} diff --git a/packages/astro/src/compiler/markdown/micromark-encode.ts b/packages/astro/src/compiler/markdown/micromark-encode.ts new file mode 100644 index 000000000..635ab3b54 --- /dev/null +++ b/packages/astro/src/compiler/markdown/micromark-encode.ts @@ -0,0 +1,36 @@ +import type { Token } from 'micromark/dist/shared-types'; +import type { MicromarkExtension, MicromarkExtensionContext } from '../../@types/micromark'; + +const characterReferences = { + '"': 'quot', + '&': 'amp', + '<': 'lt', + '>': 'gt', + '{': 'lbrace', + '}': 'rbrace', +}; + +type EncodedChars = '"' | '&' | '<' | '>' | '{' | '}'; + +/** Encode HTML entity */ +function encode(value: string): string { + return value.replace(/["&<>{}]/g, (raw: string) => { + return '&' + characterReferences[raw as EncodedChars] + ';'; + }); +} + +/** Encode Markdown node */ +function encodeToken(this: MicromarkExtensionContext) { + const token: Token = arguments[0]; + const value = this.sliceSerialize(token); + this.raw(encode(value)); +} + +const plugin: MicromarkExtension = { + exit: { + codeTextData: encodeToken, + codeFlowValue: encodeToken, + }, +}; + +export { plugin as encodeMarkdown }; diff --git a/packages/astro/src/compiler/markdown/micromark-mdx-astro.ts b/packages/astro/src/compiler/markdown/micromark-mdx-astro.ts new file mode 100644 index 000000000..b978ad407 --- /dev/null +++ b/packages/astro/src/compiler/markdown/micromark-mdx-astro.ts @@ -0,0 +1,22 @@ +import type { MicromarkExtension } from '../../@types/micromark'; +import mdxExpression from 'micromark-extension-mdx-expression'; +import mdxJsx from 'micromark-extension-mdx-jsx'; + +/** + * Keep MDX. + */ +export function encodeAstroMdx() { + const extension: MicromarkExtension = { + enter: { + mdxJsxFlowTag(node: any) { + const mdx = this.sliceSerialize(node); + this.raw(mdx); + }, + }, + }; + + return { + htmlAstro: [mdxExpression(), mdxJsx()], + mdAstro: extension, + }; +} diff --git a/packages/astro/src/compiler/markdown/micromark.d.ts b/packages/astro/src/compiler/markdown/micromark.d.ts new file mode 100644 index 000000000..fd094306e --- /dev/null +++ b/packages/astro/src/compiler/markdown/micromark.d.ts @@ -0,0 +1,11 @@ +declare module 'micromark-extension-mdx-expression' { + import type { HtmlExtension } from 'micromark/dist/shared-types'; + + export default function (): HtmlExtension; +} + +declare module 'micromark-extension-mdx-jsx' { + import type { HtmlExtension } from 'micromark/dist/shared-types'; + + export default function (): HtmlExtension; +} diff --git a/packages/astro/src/compiler/transform/doctype.ts b/packages/astro/src/compiler/transform/doctype.ts new file mode 100644 index 000000000..e871f5b48 --- /dev/null +++ b/packages/astro/src/compiler/transform/doctype.ts @@ -0,0 +1,36 @@ +import { Transformer } from '../../@types/transformer'; + +/** Transform <!doctype> tg */ +export default function (_opts: { filename: string; fileID: string }): Transformer { + let hasDoctype = false; + + return { + visitors: { + html: { + Element: { + enter(node, parent, _key, index) { + if (node.name === '!doctype') { + hasDoctype = true; + } + if (node.name === 'html' && !hasDoctype) { + const dtNode = { + start: 0, + end: 0, + attributes: [{ type: 'Attribute', name: 'html', value: true, start: 0, end: 0 }], + children: [], + name: '!doctype', + type: 'Element', + }; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + parent.children!.splice(index, 0, dtNode); + hasDoctype = true; + } + }, + }, + }, + }, + async finalize() { + // Nothing happening here. + }, + }; +} diff --git a/packages/astro/src/compiler/transform/index.ts b/packages/astro/src/compiler/transform/index.ts new file mode 100644 index 000000000..2fe0d0e73 --- /dev/null +++ b/packages/astro/src/compiler/transform/index.ts @@ -0,0 +1,100 @@ +import type { Ast, TemplateNode } from 'astro-parser'; +import type { NodeVisitor, TransformOptions, Transformer, VisitorFn } from '../../@types/transformer'; + +import { walk } from 'estree-walker'; + +// Transformers +import transformStyles from './styles.js'; +import transformDoctype from './doctype.js'; +import transformModuleScripts from './module-scripts.js'; +import transformCodeBlocks from './prism.js'; + +interface VisitorCollection { + enter: Map<string, VisitorFn[]>; + leave: Map<string, VisitorFn[]>; +} + +/** Add visitors to given collection */ +function addVisitor(visitor: NodeVisitor, collection: VisitorCollection, nodeName: string, event: 'enter' | 'leave') { + if (typeof visitor[event] !== 'function') return; + if (!collection[event]) collection[event] = new Map<string, VisitorFn[]>(); + + const visitors = collection[event].get(nodeName) || []; + visitors.push(visitor[event] as any); + collection[event].set(nodeName, visitors); +} + +/** Compile visitor actions from transformer */ +function collectVisitors(transformer: Transformer, htmlVisitors: VisitorCollection, cssVisitors: VisitorCollection, finalizers: Array<() => Promise<void>>) { + if (transformer.visitors) { + if (transformer.visitors.html) { + for (const [nodeName, visitor] of Object.entries(transformer.visitors.html)) { + addVisitor(visitor, htmlVisitors, nodeName, 'enter'); + addVisitor(visitor, htmlVisitors, nodeName, 'leave'); + } + } + if (transformer.visitors.css) { + for (const [nodeName, visitor] of Object.entries(transformer.visitors.css)) { + addVisitor(visitor, cssVisitors, nodeName, 'enter'); + addVisitor(visitor, cssVisitors, nodeName, 'leave'); + } + } + } + finalizers.push(transformer.finalize); +} + +/** Utility for formatting visitors */ +function createVisitorCollection() { + return { + enter: new Map<string, VisitorFn[]>(), + leave: new Map<string, VisitorFn[]>(), + }; +} + +/** Walk AST with collected visitors */ +function walkAstWithVisitors(tmpl: TemplateNode, collection: VisitorCollection) { + walk(tmpl, { + enter(node, parent, key, index) { + if (collection.enter.has(node.type)) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const fns = collection.enter.get(node.type)!; + for (let fn of fns) { + fn.call(this, node, parent, key, index); + } + } + }, + leave(node, parent, key, index) { + if (collection.leave.has(node.type)) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const fns = collection.leave.get(node.type)!; + for (let fn of fns) { + fn.call(this, node, parent, key, index); + } + } + }, + }); +} + +/** + * Transform + * Step 2/3 in Astro SSR. + * Transform is the point at which we mutate the AST before sending off to + * Codegen, and then to Snowpack. In some ways, it‘s a preprocessor. + */ +export async function transform(ast: Ast, opts: TransformOptions) { + const htmlVisitors = createVisitorCollection(); + const cssVisitors = createVisitorCollection(); + const finalizers: Array<() => Promise<void>> = []; + + const optimizers = [transformStyles(opts), transformDoctype(opts), transformModuleScripts(opts), transformCodeBlocks(ast.module)]; + + for (const optimizer of optimizers) { + collectVisitors(optimizer, htmlVisitors, cssVisitors, finalizers); + } + + walkAstWithVisitors(ast.css, cssVisitors); + walkAstWithVisitors(ast.html, htmlVisitors); + + // Run all of the finalizer functions in parallel because why not. + await Promise.all(finalizers.map((fn) => fn())); +} diff --git a/packages/astro/src/compiler/transform/module-scripts.ts b/packages/astro/src/compiler/transform/module-scripts.ts new file mode 100644 index 000000000..aff1ec4f6 --- /dev/null +++ b/packages/astro/src/compiler/transform/module-scripts.ts @@ -0,0 +1,43 @@ +import type { Transformer } from '../../@types/transformer'; +import type { CompileOptions } from '../../@types/compiler'; + +import path from 'path'; +import { getAttrValue, setAttrValue } from '../../ast.js'; + +/** Transform <script type="module"> */ +export default function ({ compileOptions, filename }: { compileOptions: CompileOptions; filename: string; fileID: string }): Transformer { + const { astroConfig } = compileOptions; + const { astroRoot } = astroConfig; + const fileUrl = new URL(`file://${filename}`); + + return { + visitors: { + html: { + Element: { + enter(node) { + let name = node.name; + if (name !== 'script') { + return; + } + + let type = getAttrValue(node.attributes, 'type'); + if (type !== 'module') { + return; + } + + let src = getAttrValue(node.attributes, 'src'); + if (!src || !src.startsWith('.')) { + return; + } + + const srcUrl = new URL(src, fileUrl); + const fromAstroRoot = path.posix.relative(astroRoot.pathname, srcUrl.pathname); + const absoluteUrl = `/_astro/${fromAstroRoot}`; + setAttrValue(node.attributes, 'src', absoluteUrl); + }, + }, + }, + }, + async finalize() {}, + }; +} diff --git a/packages/astro/src/compiler/transform/postcss-scoped-styles/index.ts b/packages/astro/src/compiler/transform/postcss-scoped-styles/index.ts new file mode 100644 index 000000000..23350869c --- /dev/null +++ b/packages/astro/src/compiler/transform/postcss-scoped-styles/index.ts @@ -0,0 +1,106 @@ +import { Declaration, Plugin } from 'postcss'; + +interface AstroScopedOptions { + className: string; +} + +interface Selector { + start: number; + end: number; + value: string; +} + +const CSS_SEPARATORS = new Set([' ', ',', '+', '>', '~']); +const KEYFRAME_PERCENT = /\d+\.?\d*%/; + +/** HTML tags that should never get scoped classes */ +export const NEVER_SCOPED_TAGS = new Set<string>(['base', 'body', 'font', 'frame', 'frameset', 'head', 'html', 'link', 'meta', 'noframes', 'noscript', 'script', 'style', 'title']); + +/** + * Scope Rules + * Given a selector string (`.btn>span,.nav>span`), add an additional CSS class to every selector (`.btn.myClass>span.myClass,.nav.myClass>span.myClass`) + * @param {string} selector The minified selector string to parse. Cannot contain arbitrary whitespace (other than child selector syntax). + * @param {string} className The CSS class to apply. + */ +export function scopeRule(selector: string, className: string) { + // if this is a keyframe keyword, return original selector + if (selector === 'from' || selector === 'to' || KEYFRAME_PERCENT.test(selector)) { + return selector; + } + + // For everything else, parse & scope + const c = className.replace(/^\.?/, '.'); // make sure class always has leading '.' + const selectors: Selector[] = []; + let ss = selector; // final output + + // Pass 1: parse selector string; extract top-level selectors + { + let start = 0; + let lastValue = ''; + let parensOpen = false; + for (let n = 0; n < ss.length; n++) { + const isEnd = n === selector.length - 1; + if (selector[n] === '(') parensOpen = true; + if (selector[n] === ')') parensOpen = false; + if (isEnd || (parensOpen === false && CSS_SEPARATORS.has(selector[n]))) { + lastValue = selector.substring(start, isEnd ? undefined : n); + if (!lastValue) continue; + selectors.push({ start, end: isEnd ? n + 1 : n, value: lastValue }); + start = n + 1; + } + } + } + + // Pass 2: starting from end, transform selectors w/ scoped class + for (let i = selectors.length - 1; i >= 0; i--) { + const { start, end, value } = selectors[i]; + const head = ss.substring(0, start); + const tail = ss.substring(end); + + // replace '*' with className + if (value === '*') { + ss = head + c + tail; + continue; + } + + // leave :global() alone! + if (value.startsWith(':global(')) { + ss = + head + + ss + .substring(start, end) + .replace(/^:global\(/, '') + .replace(/\)$/, '') + + tail; + continue; + } + + // don‘t scope body, title, etc. + if (NEVER_SCOPED_TAGS.has(value)) { + ss = head + value + tail; + continue; + } + + // scope everything else + let newSelector = ss.substring(start, end); + const pseudoIndex = newSelector.indexOf(':'); + if (pseudoIndex > 0) { + // if there‘s a pseudoclass (:focus) + ss = head + newSelector.substring(start, pseudoIndex) + c + newSelector.substr(pseudoIndex) + tail; + } else { + ss = head + newSelector + c + tail; + } + } + + return ss; +} + +/** PostCSS Scope plugin */ +export default function astroScopedStyles(options: AstroScopedOptions): Plugin { + return { + postcssPlugin: '@astro/postcss-scoped-styles', + Rule(rule) { + rule.selector = scopeRule(rule.selector, options.className); + }, + }; +} diff --git a/packages/astro/src/compiler/transform/prism.ts b/packages/astro/src/compiler/transform/prism.ts new file mode 100644 index 000000000..6f8eb5bba --- /dev/null +++ b/packages/astro/src/compiler/transform/prism.ts @@ -0,0 +1,89 @@ +import type { Transformer } from '../../@types/transformer'; +import type { Script } from 'astro-parser'; +import { getAttrValue } from '../../ast.js'; + +const PRISM_IMPORT = `import Prism from 'astro/components/Prism.astro';\n`; +const prismImportExp = /import Prism from ['"]astro\/components\/Prism.astro['"]/; +/** escaping code samples that contain template string replacement parts, ${foo} or example. */ +function escape(code: string) { + return code.replace(/[`$]/g, (match) => { + return '\\' + match; + }); +} +/** default export - Transform prism */ +export default function (module: Script): Transformer { + let usesPrism = false; + + return { + visitors: { + html: { + Element: { + enter(node) { + if (node.name !== 'code') return; + const className = getAttrValue(node.attributes, 'class') || ''; + const classes = className.split(' '); + + let lang; + for (let cn of classes) { + const matches = /language-(.+)/.exec(cn); + if (matches) { + lang = matches[1]; + } + } + + if (!lang) return; + + let code; + if (node.children?.length) { + code = node.children[0].data; + } + + const repl = { + start: 0, + end: 0, + type: 'InlineComponent', + name: 'Prism', + attributes: [ + { + type: 'Attribute', + name: 'lang', + value: [ + { + type: 'Text', + raw: lang, + data: lang, + }, + ], + }, + { + type: 'Attribute', + name: 'code', + value: [ + { + type: 'MustacheTag', + expression: { + type: 'Expression', + codeChunks: ['`' + escape(code) + '`'], + children: [], + }, + }, + ], + }, + ], + children: [], + }; + + this.replace(repl); + usesPrism = true; + }, + }, + }, + }, + async finalize() { + // Add the Prism import if needed. + if (usesPrism && !prismImportExp.test(module.content)) { + module.content = PRISM_IMPORT + module.content; + } + }, + }; +} diff --git a/packages/astro/src/compiler/transform/styles.ts b/packages/astro/src/compiler/transform/styles.ts new file mode 100644 index 000000000..efabf11fe --- /dev/null +++ b/packages/astro/src/compiler/transform/styles.ts @@ -0,0 +1,290 @@ +import crypto from 'crypto'; +import fs from 'fs'; +import { createRequire } from 'module'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import autoprefixer from 'autoprefixer'; +import postcss, { Plugin } from 'postcss'; +import postcssKeyframes from 'postcss-icss-keyframes'; +import findUp from 'find-up'; +import sass from 'sass'; +import type { RuntimeMode } from '../../@types/astro'; +import type { TransformOptions, Transformer } from '../../@types/transformer'; +import type { TemplateNode } from 'astro-parser'; +import { debug } from '../../logger.js'; +import astroScopedStyles, { NEVER_SCOPED_TAGS } from './postcss-scoped-styles/index.js'; + +type StyleType = 'css' | 'scss' | 'sass' | 'postcss'; + +declare global { + interface ImportMeta { + /** https://nodejs.org/api/esm.html#esm_import_meta_resolve_specifier_parent */ + resolve(specifier: string, parent?: string): Promise<any>; + } +} + +const getStyleType: Map<string, StyleType> = new Map([ + ['.css', 'css'], + ['.pcss', 'postcss'], + ['.sass', 'sass'], + ['.scss', 'scss'], + ['css', 'css'], + ['sass', 'sass'], + ['scss', 'scss'], + ['text/css', 'css'], + ['text/sass', 'sass'], + ['text/scss', 'scss'], +]); + +/** Should be deterministic, given a unique filename */ +function hashFromFilename(filename: string): string { + const hash = crypto.createHash('sha256'); + return hash + .update(filename.replace(/\\/g, '/')) + .digest('base64') + .toString() + .replace(/[^A-Za-z0-9-]/g, '') + .substr(0, 8); +} + +export interface StyleTransformResult { + css: string; + type: StyleType; +} + +interface StylesMiniCache { + nodeModules: Map<string, string>; // filename: node_modules location + tailwindEnabled?: boolean; // cache once per-run +} + +/** Simple cache that only exists in memory per-run. Prevents the same lookups from happening over and over again within the same build or dev server session. */ +const miniCache: StylesMiniCache = { + nodeModules: new Map<string, string>(), +}; + +export interface TransformStyleOptions { + type?: string; + filename: string; + scopedClass: string; + mode: RuntimeMode; +} + +/** given a class="" string, does it contain a given class? */ +function hasClass(classList: string, className: string): boolean { + if (!className) return false; + for (const c of classList.split(' ')) { + if (className === c.trim()) return true; + } + return false; +} + +/** Convert styles to scoped CSS */ +async function transformStyle(code: string, { type, filename, scopedClass, mode }: TransformStyleOptions): Promise<StyleTransformResult> { + let styleType: StyleType = 'css'; // important: assume CSS as default + if (type) { + styleType = getStyleType.get(type) || styleType; + } + + // add file path to includePaths + let includePaths: string[] = [path.dirname(filename)]; + + // include node_modules to includePaths (allows @use-ing node modules, if it can be located) + const cachedNodeModulesDir = miniCache.nodeModules.get(filename); + if (cachedNodeModulesDir) { + includePaths.push(cachedNodeModulesDir); + } else { + const nodeModulesDir = await findUp('node_modules', { type: 'directory', cwd: path.dirname(filename) }); + if (nodeModulesDir) { + miniCache.nodeModules.set(filename, nodeModulesDir); + includePaths.push(nodeModulesDir); + } + } + + // 1. Preprocess (currently only Sass supported) + let css = ''; + switch (styleType) { + case 'css': { + css = code; + break; + } + case 'sass': + case 'scss': { + css = sass.renderSync({ data: code, includePaths }).css.toString('utf8'); + break; + } + default: { + throw new Error(`Unsupported: <style lang="${styleType}">`); + } + } + + // 2. Post-process (PostCSS) + const postcssPlugins: Plugin[] = []; + + // 2a. Tailwind (only if project uses Tailwind) + if (miniCache.tailwindEnabled) { + try { + const require = createRequire(import.meta.url); + const tw = require.resolve('tailwindcss', { paths: [import.meta.url, process.cwd()] }); + postcssPlugins.push(require(tw) as any); + } catch (err) { + // eslint-disable-next-line no-console + console.error(err); + throw new Error(`tailwindcss not installed. Try running \`npm install tailwindcss\` and trying again.`); + } + } + + // 2b. Astro scoped styles (always on) + postcssPlugins.push(astroScopedStyles({ className: scopedClass })); + + // 2c. Scoped @keyframes + postcssPlugins.push( + postcssKeyframes({ + generateScopedName(keyframesName) { + return `${keyframesName}-${scopedClass}`; + }, + }) + ); + + // 2d. Autoprefixer (always on) + postcssPlugins.push(autoprefixer()); + + // 2e. Run PostCSS + css = await postcss(postcssPlugins) + .process(css, { from: filename, to: undefined }) + .then((result) => result.css); + + return { css, type: styleType }; +} + +/** Transform <style> tags */ +export default function transformStyles({ compileOptions, filename, fileID }: TransformOptions): Transformer { + const styleNodes: TemplateNode[] = []; // <style> tags to be updated + const styleTransformPromises: Promise<StyleTransformResult>[] = []; // async style transform results to be finished in finalize(); + const scopedClass = `astro-${hashFromFilename(fileID)}`; // this *should* generate same hash from fileID every time + + // find Tailwind config, if first run (cache for subsequent runs) + if (miniCache.tailwindEnabled === undefined) { + const tailwindNames = ['tailwind.config.js', 'tailwind.config.mjs']; + for (const loc of tailwindNames) { + const tailwindLoc = path.join(fileURLToPath(compileOptions.astroConfig.projectRoot), loc); + if (fs.existsSync(tailwindLoc)) { + miniCache.tailwindEnabled = true; // Success! We have a Tailwind config file. + debug(compileOptions.logging, 'tailwind', 'Found config. Enabling.'); + break; + } + } + if (miniCache.tailwindEnabled !== true) miniCache.tailwindEnabled = false; // We couldn‘t find one; mark as false + debug(compileOptions.logging, 'tailwind', 'No config found. Skipping.'); + } + + return { + visitors: { + html: { + Element: { + enter(node) { + // 1. if <style> tag, transform it and continue to next node + if (node.name === 'style') { + // Same as ast.css (below) + const code = Array.isArray(node.children) ? node.children.map(({ data }: any) => data).join('\n') : ''; + if (!code) return; + const langAttr = (node.attributes || []).find(({ name }: any) => name === 'lang'); + styleNodes.push(node); + styleTransformPromises.push( + transformStyle(code, { + type: (langAttr && langAttr.value[0] && langAttr.value[0].data) || undefined, + filename, + scopedClass, + mode: compileOptions.mode, + }) + ); + return; + } + + // 2. add scoped HTML classes + if (NEVER_SCOPED_TAGS.has(node.name)) return; // only continue if this is NOT a <script> tag, etc. + // Note: currently we _do_ scope web components/custom elements. This seems correct? + + if (!node.attributes) node.attributes = []; + const classIndex = node.attributes.findIndex(({ name }: any) => name === 'class'); + if (classIndex === -1) { + // 3a. element has no class="" attribute; add one and append scopedClass + node.attributes.push({ start: -1, end: -1, type: 'Attribute', name: 'class', value: [{ type: 'Text', raw: scopedClass, data: scopedClass }] }); + } else { + // 3b. element has class=""; append scopedClass + const attr = node.attributes[classIndex]; + for (let k = 0; k < attr.value.length; k++) { + if (attr.value[k].type === 'Text') { + // don‘t add same scopedClass twice + if (!hasClass(attr.value[k].data, scopedClass)) { + // string literal + attr.value[k].raw += ' ' + scopedClass; + attr.value[k].data += ' ' + scopedClass; + } + } else if (attr.value[k].type === 'MustacheTag' && attr.value[k]) { + // don‘t add same scopedClass twice (this check is a little more basic, but should suffice) + if (!attr.value[k].expression.codeChunks[0].includes(`' ${scopedClass}'`)) { + // MustacheTag + // FIXME: this won't work when JSX element can appear in attributes (rare but possible). + attr.value[k].expression.codeChunks[0] = `(${attr.value[k].expression.codeChunks[0]}) + ' ${scopedClass}'`; + } + } + } + } + }, + }, + }, + // CSS: compile styles, apply CSS Modules scoping + css: { + Style: { + enter(node) { + // Same as ast.html (above) + // Note: this is duplicated from html because of the compiler we‘re using; in a future version we should combine these + if (!node.content || !node.content.styles) return; + const code = node.content.styles; + const langAttr = (node.attributes || []).find(({ name }: any) => name === 'lang'); + styleNodes.push(node); + styleTransformPromises.push( + transformStyle(code, { + type: (langAttr && langAttr.value[0] && langAttr.value[0].data) || undefined, + filename, + scopedClass, + mode: compileOptions.mode, + }) + ); + }, + }, + }, + }, + async finalize() { + const styleTransforms = await Promise.all(styleTransformPromises); + + styleTransforms.forEach((result, n) => { + if (styleNodes[n].attributes) { + // 1. Replace with final CSS + const isHeadStyle = !styleNodes[n].content; + if (isHeadStyle) { + // Note: <style> tags in <head> have different attributes/rules, because of the parser. Unknown why + (styleNodes[n].children as any) = [{ ...(styleNodes[n].children as any)[0], data: result.css }]; + } else { + styleNodes[n].content.styles = result.css; + } + + // 2. Update <style> attributes + const styleTypeIndex = styleNodes[n].attributes.findIndex(({ name }: any) => name === 'type'); + // add type="text/css" + if (styleTypeIndex !== -1) { + styleNodes[n].attributes[styleTypeIndex].value[0].raw = 'text/css'; + styleNodes[n].attributes[styleTypeIndex].value[0].data = 'text/css'; + } else { + styleNodes[n].attributes.push({ name: 'type', type: 'Attribute', value: [{ type: 'Text', raw: 'text/css', data: 'text/css' }] }); + } + // remove lang="*" + const styleLangIndex = styleNodes[n].attributes.findIndex(({ name }: any) => name === 'lang'); + if (styleLangIndex !== -1) styleNodes[n].attributes.splice(styleLangIndex, 1); + // TODO: add data-astro for later + // styleNodes[n].attributes.push({ name: 'data-astro', type: 'Attribute', value: true }); + } + }); + }, + }; +} diff --git a/packages/astro/src/config.ts b/packages/astro/src/config.ts new file mode 100644 index 000000000..8f3ebaf5a --- /dev/null +++ b/packages/astro/src/config.ts @@ -0,0 +1,85 @@ +import type { AstroConfig } from './@types/astro'; +import { join as pathJoin, resolve as pathResolve } from 'path'; +import { existsSync } from 'fs'; + +/** Type util */ +const type = (thing: any): string => (Array.isArray(thing) ? 'Array' : typeof thing); + +/** Throws error if a user provided an invalid config. Manually-implemented to avoid a heavy validation library. */ +function validateConfig(config: any): void { + // basic + if (config === undefined || config === null) throw new Error(`[astro config] Config empty!`); + if (typeof config !== 'object') throw new Error(`[astro config] Expected object, received ${typeof config}`); + + // strings + for (const key of ['projectRoot', 'astroRoot', 'dist', 'public', 'site']) { + if (config[key] !== undefined && config[key] !== null && typeof config[key] !== 'string') { + throw new Error(`[astro config] ${key}: ${JSON.stringify(config[key])}\n Expected string, received ${type(config[key])}.`); + } + } + + // booleans + for (const key of ['sitemap']) { + if (config[key] !== undefined && config[key] !== null && typeof config[key] !== 'boolean') { + throw new Error(`[astro config] ${key}: ${JSON.stringify(config[key])}\n Expected boolean, received ${type(config[key])}.`); + } + } + + if(typeof config.devOptions?.port !== 'number') { + throw new Error(`[astro config] devOptions.port: Expected number, received ${type(config.devOptions?.port)}`) + } +} + +/** Set default config values */ +function configDefaults(userConfig?: any): any { + const config: any = { ...(userConfig || {}) }; + + if (!config.projectRoot) config.projectRoot = '.'; + if (!config.astroRoot) config.astroRoot = './src'; + if (!config.dist) config.dist = './dist'; + if (!config.public) config.public = './public'; + if (!config.devOptions) config.devOptions = {}; + if (!config.devOptions.port) config.devOptions.port = 3000; + if (!config.buildOptions) config.buildOptions = {}; + if (typeof config.buildOptions.sitemap === 'undefined') config.buildOptions.sitemap = true; + + return config; +} + +/** Turn raw config values into normalized values */ +function normalizeConfig(userConfig: any, root: string): AstroConfig { + const config: any = { ...(userConfig || {}) }; + + const fileProtocolRoot = `file://${root}/`; + config.projectRoot = new URL(config.projectRoot + '/', fileProtocolRoot); + config.astroRoot = new URL(config.astroRoot + '/', fileProtocolRoot); + config.public = new URL(config.public + '/', fileProtocolRoot); + + return config as AstroConfig; +} + +/** Attempt to load an `astro.config.mjs` file */ +export async function loadConfig(rawRoot: string | undefined, configFileName = 'astro.config.mjs'): Promise<AstroConfig> { + if (typeof rawRoot === 'undefined') { + rawRoot = process.cwd(); + } + + const root = pathResolve(rawRoot); + const astroConfigPath = pathJoin(root, configFileName); + + // load + let config: any; + if (existsSync(astroConfigPath)) { + config = configDefaults((await import(astroConfigPath)).default); + } else { + config = configDefaults(); + } + + // validate + validateConfig(config); + + // normalize + config = normalizeConfig(config, root); + + return config as AstroConfig; +} diff --git a/packages/astro/src/dev.ts b/packages/astro/src/dev.ts new file mode 100644 index 000000000..4ca8e28e9 --- /dev/null +++ b/packages/astro/src/dev.ts @@ -0,0 +1,97 @@ +import type { AstroConfig } from './@types/astro'; +import type { LogOptions } from './logger.js'; + +import { logger as snowpackLogger } from 'snowpack'; +import { bold, green } from 'kleur/colors'; +import http from 'http'; +import { relative as pathRelative } from 'path'; +import { performance } from 'perf_hooks'; +import { defaultLogDestination, error, info, parseError } from './logger.js'; +import { createRuntime } from './runtime.js'; + +const hostname = '127.0.0.1'; + +// Disable snowpack from writing to stdout/err. +snowpackLogger.level = 'silent'; + +const logging: LogOptions = { + level: 'debug', + dest: defaultLogDestination, +}; + +/** The primary dev action */ +export default async function dev(astroConfig: AstroConfig) { + const startServerTime = performance.now(); + const { projectRoot } = astroConfig; + + const runtime = await createRuntime(astroConfig, { mode: 'development', logging }); + + const server = http.createServer(async (req, res) => { + const result = await runtime.load(req.url); + + switch (result.statusCode) { + case 200: { + if (result.contentType) { + res.setHeader('Content-Type', result.contentType); + } + res.statusCode = 200; + res.write(result.contents); + res.end(); + break; + } + case 404: { + const fullurl = new URL(req.url || '/', 'https://example.org/'); + const reqPath = decodeURI(fullurl.pathname); + error(logging, 'static', 'Not found', reqPath); + res.statusCode = 404; + + const fourOhFourResult = await runtime.load('/404'); + if (fourOhFourResult.statusCode === 200) { + if (fourOhFourResult.contentType) { + res.setHeader('Content-Type', fourOhFourResult.contentType); + } + res.write(fourOhFourResult.contents); + } else { + res.setHeader('Content-Type', 'text/plain'); + res.write('Not Found'); + } + res.end(); + break; + } + case 500: { + switch (result.type) { + case 'parse-error': { + const err = result.error; + err.filename = pathRelative(projectRoot.pathname, err.filename); + parseError(logging, err); + break; + } + default: { + error(logging, 'executing astro', result.error); + break; + } + } + res.statusCode = 500; + + let errorResult = await runtime.load(`/500?error=${encodeURIComponent(result.error.stack || result.error.toString())}`); + if(errorResult.statusCode === 200) { + if (errorResult.contentType) { + res.setHeader('Content-Type', errorResult.contentType); + } + res.write(errorResult.contents); + } else { + res.write(result.error.toString()); + } + res.end(); + break; + } + } + }); + + const port = astroConfig.devOptions.port; + server.listen(port, hostname, () => { + const endServerTime = performance.now(); + info(logging, 'dev server', green(`Server started in ${Math.floor(endServerTime - startServerTime)}ms.`)); + info(logging, 'dev server', `${green('Local:')} http://${hostname}:${port}/`); + }); +} diff --git a/packages/astro/src/frontend/500.astro b/packages/astro/src/frontend/500.astro new file mode 100644 index 000000000..01fab8bea --- /dev/null +++ b/packages/astro/src/frontend/500.astro @@ -0,0 +1,128 @@ +--- +import Prism from 'astro/components/Prism.astro'; +let title = 'Uh oh...'; + +const error = Astro.request.url.searchParams.get('error'); +--- + +<!doctype html> +<html lang="en"> + <head> + <title>Error 500</title> + <link rel="preconnect"href="https://fonts.gstatic.com"> + <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono&family=IBM+Plex+Sans&display=swap"> + <link rel="stylesheet" href="http://cdn.skypack.dev/prism-themes/themes/prism-material-dark.css"> + + <style> + * { + box-sizing: border-box; + margin: 0; + } + + :global(:root) { + --font-sans: "IBM Plex Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans", "Helvetica Neue", sans-serif; + --font-mono: "IBM Plex Mono", Consolas, "Andale Mono WT", "Andale Mono", + "Lucida Console", "Lucida Sans Typewriter", "DejaVu Sans Mono", + "Bitstream Vera Sans Mono", "Liberation Mono", "Nimbus Mono L", Monaco, + "Courier New", Courier, monospace; + --color-gray-800: #1F2937; + --color-gray-500: #6B7280; + --color-gray-400: #9CA3AF; + --color-gray-100: #F3F4F6; + --color-red: #FF1639; + } + + html, body { + width: 100vw; + height: 100%; + min-height: 100vh; + + font-family: var(--font-sans); + font-weight: 400; + background: var(--color-gray-100); + text-align: center; + } + + body { + display: grid; + place-content: center; + } + + header { + display: flex; + flex-direction: column; + align-items: center; + font-family: var(--font-sans); + font-size: 2.5rem; + font-size: clamp(24px, calc(2vw + 1rem), 2.5rem); + } + + header h1 { + margin: 0.25em; + margin-right: 0; + font-weight: 400; + letter-spacing: -2px; + line-height: 1; + } + + header h1 .title { + color: var(--color-gray-400); + white-space: nowrap; + } + + header svg { + margin-bottom: -0.125em; + color: var(--color-red); + } + + p { + font-size: 1.75rem; + font-size: clamp(14px, calc(2vw + 0.5rem), 1.75rem); + flex: 1; + } + + .error-message { + display: grid; + justify-content: center; + margin-top: 4rem; + } + + .error-message :global(code[class*="language-"]) { + background: var(--color-gray-800); + } + .error-message :global(pre) { + margin: 0; + font-family: var(--font-mono); + font-size: 0.85rem; + background: var(--color-gray-800); + border-radius: 8px; + } + + .error-message :global(.token.punctuation) { + color: var(--color-gray-400); + } + + .error-message :global(.token.operator) { + color: var(--color-gray-400); + } + </style> + </head> + <body> + <main> + <header> + <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor" width="1.75em" height="1.75em"> + <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" /> + </svg> + <h1><span class="error">500 Error </span><span class="title">{title}</span></h1> + </header> + + <article> + <p>Astro had some trouble loading this page.</p> + + <div class="error-message"> + <Prism lang="shell" code={error} /> + </div> + </article> + </main> + </body> +</html> diff --git a/packages/astro/src/frontend/SvelteWrapper.svelte b/packages/astro/src/frontend/SvelteWrapper.svelte new file mode 100644 index 000000000..eb4cbb7d9 --- /dev/null +++ b/packages/astro/src/frontend/SvelteWrapper.svelte @@ -0,0 +1,7 @@ +<script> +const { __astro_component: Component, __astro_children, ...props } = $$props; +</script> + +<Component {...props}> + {@html __astro_children} +</Component> diff --git a/packages/astro/src/frontend/SvelteWrapper.svelte.client.ts b/packages/astro/src/frontend/SvelteWrapper.svelte.client.ts new file mode 100644 index 000000000..9df168895 --- /dev/null +++ b/packages/astro/src/frontend/SvelteWrapper.svelte.client.ts @@ -0,0 +1,166 @@ +/* eslint-disable */ +// @ts-nocheck +// TODO: don't precompile this, but it works for now +import { + HtmlTag, + SvelteComponentDev, + assign, + claim_component, + create_component, + destroy_component, + detach_dev, + dispatch_dev, + empty, + exclude_internal_props, + get_spread_object, + get_spread_update, + init, + insert_dev, + mount_component, + noop, + not_equal, + transition_in, + transition_out, + validate_slots, +} from 'svelte/internal'; + +const file = 'App.svelte'; + +// (5:0) <Component {...props}> +function create_default_slot(ctx) { + let html_tag; + let html_anchor; + + const block = { + c: function create() { + html_anchor = empty(); + this.h(); + }, + l: function claim(nodes) { + html_anchor = empty(); + this.h(); + }, + h: function hydrate() { + html_tag = new HtmlTag(html_anchor); + }, + m: function mount(target, anchor) { + html_tag.m(/*__astro_children*/ ctx[1], target, anchor); + insert_dev(target, html_anchor, anchor); + }, + p: noop, + d: function destroy(detaching) { + if (detaching) detach_dev(html_anchor); + if (detaching) html_tag.d(); + }, + }; + + dispatch_dev('SvelteRegisterBlock', { + block, + id: create_default_slot.name, + type: 'slot', + source: '(5:0) <Component {...props}>', + ctx, + }); + + return block; +} + +function create_fragment(ctx) { + let component; + let current; + const component_spread_levels = [/*props*/ ctx[2]]; + + let component_props = { + $$slots: { default: [create_default_slot] }, + $$scope: { ctx }, + }; + + for (let i = 0; i < component_spread_levels.length; i += 1) { + component_props = assign(component_props, component_spread_levels[i]); + } + + component = new /*Component*/ ctx[0]({ props: component_props, $$inline: true }); + + const block = { + c: function create() { + create_component(component.$$.fragment); + }, + l: function claim(nodes) { + claim_component(component.$$.fragment, nodes); + }, + m: function mount(target, anchor) { + mount_component(component, target, anchor); + current = true; + }, + p: function update(ctx, [dirty]) { + const component_changes = dirty & /*props*/ 4 ? get_spread_update(component_spread_levels, [get_spread_object(/*props*/ ctx[2])]) : {}; + + if (dirty & /*$$scope*/ 16) { + component_changes.$$scope = { dirty, ctx }; + } + + component.$set(component_changes); + }, + i: function intro(local) { + if (current) return; + transition_in(component.$$.fragment, local); + current = true; + }, + o: function outro(local) { + transition_out(component.$$.fragment, local); + current = false; + }, + d: function destroy(detaching) { + destroy_component(component, detaching); + }, + }; + + dispatch_dev('SvelteRegisterBlock', { + block, + id: create_fragment.name, + type: 'component', + source: '', + ctx, + }); + + return block; +} + +function instance($$self, $$props, $$invalidate) { + let { $$slots: slots = {}, $$scope } = $$props; + validate_slots('App', slots, []); + const { __astro_component: Component, __astro_children, ...props } = $$props; + + $$self.$$set = ($$new_props) => { + $$invalidate(3, ($$props = assign(assign({}, $$props), exclude_internal_props($$new_props)))); + }; + + $$self.$capture_state = () => ({ Component, __astro_children, props }); + + $$self.$inject_state = ($$new_props) => { + $$invalidate(3, ($$props = assign(assign({}, $$props), $$new_props))); + }; + + if ($$props && '$$inject' in $$props) { + $$self.$inject_state($$props.$$inject); + } + + $$props = exclude_internal_props($$props); + return [Component, __astro_children, props]; +} + +class App extends SvelteComponentDev { + constructor(options) { + super(options); + init(this, options, instance, create_fragment, not_equal, {}); + + dispatch_dev('SvelteRegisterComponent', { + component: this, + tagName: 'App', + options, + id: create_fragment.name, + }); + } +} + +export default App; diff --git a/packages/astro/src/frontend/SvelteWrapper.svelte.server.ts b/packages/astro/src/frontend/SvelteWrapper.svelte.server.ts new file mode 100644 index 000000000..c5a25ff03 --- /dev/null +++ b/packages/astro/src/frontend/SvelteWrapper.svelte.server.ts @@ -0,0 +1,12 @@ +/* eslint-disable */ +// @ts-nocheck +// TODO: don't precompile this, but it works for now +/* App.svelte generated by Svelte v3.37.0 */ +import { create_ssr_component, validate_component } from 'svelte/internal'; + +const App = create_ssr_component(($$result, $$props, $$bindings, slots) => { + const { __astro_component: Component, __astro_children, ...props } = $$props; + return `${validate_component(Component, 'Component').$$render($$result, Object.assign(props), {}, { default: () => `${__astro_children}` })}`; +}); + +export default App; diff --git a/packages/astro/src/frontend/h.ts b/packages/astro/src/frontend/h.ts new file mode 100644 index 000000000..c1e21dc95 --- /dev/null +++ b/packages/astro/src/frontend/h.ts @@ -0,0 +1,65 @@ +export type HProps = Record<string, string> | null | undefined; +export type HChild = string | undefined | (() => string); +export type AstroComponent = (props: HProps, ...children: Array<HChild>) => string; +export type HTag = string | AstroComponent; + +const voidTags = new Set(['area', 'base', 'br', 'col', 'command', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'meta', 'param', 'source', 'track', 'wbr']); + +/** Generator for primary h() function */ +function* _h(tag: string, attrs: HProps, children: Array<HChild>) { + if (tag === '!doctype') { + yield '<!doctype '; + if (attrs) { + yield Object.keys(attrs).join(' '); + } + yield '>'; + return; + } + + yield `<${tag}`; + if (attrs) { + yield ' '; + for (let [key, value] of Object.entries(attrs)) { + yield `${key}="${value}"`; + } + } + yield '>'; + + // Void tags have no children. + if (voidTags.has(tag)) { + return; + } + + for (let child of children) { + // Special: If a child is a function, call it automatically. + // This lets you do {() => ...} without the extra boilerplate + // of wrapping it in a function and calling it. + if (typeof child === 'function') { + yield child(); + } else if (typeof child === 'string') { + yield child; + } else if (!child) { + // do nothing, safe to ignore falsey values. + } else { + yield child; + } + } + + yield `</${tag}>`; +} + +/** Astro‘s primary h() function. Allows it to use JSX-like syntax. */ +export async function h(tag: HTag, attrs: HProps, ...pChildren: Array<Promise<HChild>>) { + const children = await Promise.all(pChildren.flat(Infinity)); + if (typeof tag === 'function') { + // We assume it's an astro component + return tag(attrs, ...children); + } + + return Array.from(_h(tag, attrs, children)).join(''); +} + +/** Fragment helper, similar to React.Fragment */ +export function Fragment(_: HProps, ...children: Array<string>) { + return children.join(''); +} diff --git a/packages/astro/src/frontend/render/preact.ts b/packages/astro/src/frontend/render/preact.ts new file mode 100644 index 000000000..5c50b6fe3 --- /dev/null +++ b/packages/astro/src/frontend/render/preact.ts @@ -0,0 +1,31 @@ +import { h, render, ComponentType } from 'preact'; +import { renderToString } from 'preact-render-to-string'; +import { childrenToVnodes } from './utils'; +import type { ComponentRenderer } from '../../@types/renderer'; +import { createRenderer } from './renderer'; + +// This prevents tree-shaking of render. +Function.prototype(render); + +const Preact: ComponentRenderer<ComponentType> = { + jsxPragma: h, + jsxPragmaName: 'h', + renderStatic(Component) { + return async (props, ...children) => { + return renderToString(h(Component, props, childrenToVnodes(h, children))); + }; + }, + imports: { + preact: ['render', 'Fragment', 'h'], + }, + render({ Component, root, props, children }) { + return `render(h(${Component}, ${props}, h(Fragment, null, ...${children})), ${root})`; + }, +}; + +const renderer = createRenderer(Preact); + +export const __preact_static = renderer.static; +export const __preact_load = renderer.load; +export const __preact_idle = renderer.idle; +export const __preact_visible = renderer.visible; diff --git a/packages/astro/src/frontend/render/react.ts b/packages/astro/src/frontend/render/react.ts new file mode 100644 index 000000000..063c6a2b5 --- /dev/null +++ b/packages/astro/src/frontend/render/react.ts @@ -0,0 +1,32 @@ +import type { ComponentRenderer } from '../../@types/renderer'; +import React, { ComponentType } from 'react'; +import ReactDOMServer from 'react-dom/server'; +import { createRenderer } from './renderer'; +import { childrenToVnodes } from './utils'; + +// This prevents tree-shaking of render. +Function.prototype(ReactDOMServer); + +const ReactRenderer: ComponentRenderer<ComponentType> = { + jsxPragma: React.createElement, + jsxPragmaName: 'React.createElement', + renderStatic(Component) { + return async (props, ...children) => { + return ReactDOMServer.renderToString(React.createElement(Component, props, childrenToVnodes(React.createElement, children))); + }; + }, + imports: { + react: ['default: React'], + 'react-dom': ['default: ReactDOM'], + }, + render({ Component, root, children, props }) { + return `ReactDOM.hydrate(React.createElement(${Component}, ${props}, React.createElement(React.Fragment, null, ...${children})), ${root})`; + }, +}; + +const renderer = createRenderer(ReactRenderer); + +export const __react_static = renderer.static; +export const __react_load = renderer.load; +export const __react_idle = renderer.idle; +export const __react_visible = renderer.visible; diff --git a/packages/astro/src/frontend/render/renderer.ts b/packages/astro/src/frontend/render/renderer.ts new file mode 100644 index 000000000..7bdf7d8a8 --- /dev/null +++ b/packages/astro/src/frontend/render/renderer.ts @@ -0,0 +1,64 @@ +import type { DynamicRenderContext, DynamicRendererGenerator, SupportedComponentRenderer, StaticRendererGenerator } from '../../@types/renderer'; +import { childrenToH } from './utils'; + +// This prevents tree-shaking of render. +Function.prototype(childrenToH); + +/** Initialize Astro Component renderer for Static and Dynamic components */ +export function createRenderer(renderer: SupportedComponentRenderer) { + const _static: StaticRendererGenerator = (Component) => renderer.renderStatic(Component); + const _imports = (context: DynamicRenderContext) => { + const values = Object.values(renderer.imports ?? {}) + .reduce((acc, v) => { + return [...acc, `{ ${v.join(', ')} }`]; + }, []) + .join(', '); + const libs = Object.keys(renderer.imports ?? {}) + .reduce((acc: string[], lib: string) => { + return [...acc, `import("${context.frameworkUrls[lib as any]}")`]; + }, []) + .join(','); + return `const [{${context.componentExport}: Component}, ${values}] = await Promise.all([import("${context.componentUrl}")${renderer.imports ? ', ' + libs : ''}]);`; + }; + const serializeProps = ({ children: _, ...props }: Record<string, any>) => JSON.stringify(props); + const createContext = () => { + const astroId = `${Math.floor(Math.random() * 1e16)}`; + return { ['data-astro-id']: astroId, root: `document.querySelector('[data-astro-id="${astroId}"]')`, Component: 'Component' }; + }; + const createDynamicRender: DynamicRendererGenerator = (wrapperStart, wrapperEnd) => (Component, renderContext) => { + const innerContext = createContext(); + return async (props, ...children) => { + let value: string; + try { + value = await _static(Component)(props, ...children); + } catch (e) { + value = ''; + } + value = `<div data-astro-id="${innerContext['data-astro-id']}" style="display:contents">${value}</div>`; + + const script = ` + ${typeof wrapperStart === 'function' ? wrapperStart(innerContext) : wrapperStart} + ${_imports(renderContext)} + ${renderer.render({ + ...innerContext, + props: serializeProps(props), + children: `[${childrenToH(renderer, children) ?? ''}]`, + childrenAsString: `\`${children}\``, + })} + ${typeof wrapperEnd === 'function' ? wrapperEnd(innerContext) : wrapperEnd} + `; + + return [value, `<script type="module">${script.trim()}</script>`].join('\n'); + }; + }; + + return { + static: _static, + load: createDynamicRender('(async () => {', '})()'), + idle: createDynamicRender('requestIdleCallback(async () => {', '})'), + visible: createDynamicRender( + 'const o = new IntersectionObserver(async ([entry]) => { if (!entry.isIntersecting) { return; } o.disconnect();', + ({ root }) => `}); Array.from(${root}.children).forEach(child => o.observe(child))` + ), + }; +} diff --git a/packages/astro/src/frontend/render/svelte.ts b/packages/astro/src/frontend/render/svelte.ts new file mode 100644 index 000000000..13e2b8f58 --- /dev/null +++ b/packages/astro/src/frontend/render/svelte.ts @@ -0,0 +1,26 @@ +import type { ComponentRenderer } from '../../@types/renderer'; +import type { SvelteComponent } from 'svelte'; +import { createRenderer } from './renderer'; +import SvelteWrapper from '../SvelteWrapper.svelte.server'; + +const SvelteRenderer: ComponentRenderer<SvelteComponent> = { + renderStatic(Component) { + return async (props, ...children) => { + const { html } = SvelteWrapper.render({ __astro_component: Component, __astro_children: children.join('\n'), ...props }); + return html; + }; + }, + imports: { + 'svelte-runtime': ['default: render'], + }, + render({ Component, root, props, childrenAsString }) { + return `render(${root}, ${Component}, ${props}, ${childrenAsString});`; + }, +}; + +const renderer = createRenderer(SvelteRenderer); + +export const __svelte_static = renderer.static; +export const __svelte_load = renderer.load; +export const __svelte_idle = renderer.idle; +export const __svelte_visible = renderer.visible; diff --git a/packages/astro/src/frontend/render/utils.ts b/packages/astro/src/frontend/render/utils.ts new file mode 100644 index 000000000..5d13ca136 --- /dev/null +++ b/packages/astro/src/frontend/render/utils.ts @@ -0,0 +1,54 @@ +import unified from 'unified'; +import parse from 'rehype-parse'; +import toH from 'hast-to-hyperscript'; +import { ComponentRenderer } from '../../@types/renderer'; +import moize from 'moize'; +// This prevents tree-shaking of render. +Function.prototype(toH); + +/** @internal */ +function childrenToTree(children: string[]) { + return children.map((child) => (unified().use(parse, { fragment: true }).parse(child) as any).children.pop()); +} + +/** + * Converts an HTML fragment string into vnodes for rendering via provided framework + * @param h framework's `createElement` function + * @param children the HTML string children + */ +export const childrenToVnodes = moize.deep(function childrenToVnodes(h: any, children: string[]) { + const tree = childrenToTree(children); + const vnodes = tree.map((subtree) => { + if (subtree.type === 'text') return subtree.value; + return toH(h, subtree); + }); + return vnodes; +}); + +/** + * Converts an HTML fragment string into h function calls as a string + * @param h framework's `createElement` function + * @param children the HTML string children + */ +export const childrenToH = moize.deep(function childrenToH(renderer: ComponentRenderer<any>, children: string[]): any { + if (!renderer.jsxPragma) return; + const tree = childrenToTree(children); + const innerH = (name: any, attrs: Record<string, any> | null = null, _children: string[] | null = null) => { + const vnode = renderer.jsxPragma?.(name, attrs, _children); + const childStr = _children ? `, [${_children.map((child) => serializeChild(child)).join(',')}]` : ''; + /* fix(react): avoid hard-coding keys into the serialized tree */ + if (attrs && attrs.key) attrs.key = undefined; + const __SERIALIZED = `${renderer.jsxPragmaName}("${name}", ${attrs ? JSON.stringify(attrs) : 'null'}${childStr})` as string; + return { ...vnode, __SERIALIZED }; + }; + const serializeChild = (child: unknown) => { + if (['string', 'number', 'boolean'].includes(typeof child)) return JSON.stringify(child); + if (child === null) return `null`; + if ((child as any).__SERIALIZED) return (child as any).__SERIALIZED; + return innerH(child).__SERIALIZED; + }; + return tree.map((subtree) => { + if (subtree.type === 'text') return JSON.stringify(subtree.value); + return toH(innerH, subtree).__SERIALIZED + }); +}); diff --git a/packages/astro/src/frontend/render/vue.ts b/packages/astro/src/frontend/render/vue.ts new file mode 100644 index 000000000..57c3c8276 --- /dev/null +++ b/packages/astro/src/frontend/render/vue.ts @@ -0,0 +1,65 @@ +import type { ComponentRenderer } from '../../@types/renderer'; +import type { Component as VueComponent } from 'vue'; +import { renderToString } from '@vue/server-renderer'; +import { defineComponent, createSSRApp, h as createElement } from 'vue'; +import { createRenderer } from './renderer'; + +// This prevents tree-shaking of render. +Function.prototype(renderToString); + +/** + * Users might attempt to use :vueAttribute syntax to pass primitive values. + * If so, try to JSON.parse them to get the primitives + */ +function cleanPropsForVue(obj: Record<string, any>) { + let cleaned = {} as any; + for (let [key, value] of Object.entries(obj)) { + if (key.startsWith(':')) { + key = key.slice(1); + if (typeof value === 'string') { + try { + value = JSON.parse(value); + } catch (e) {} + } + } + cleaned[key] = value; + } + return cleaned; +} + +const Vue: ComponentRenderer<VueComponent> = { + jsxPragma: createElement, + jsxPragmaName: 'createElement', + renderStatic(Component) { + return async (props, ...children) => { + const App = defineComponent({ + components: { + Component, + }, + data() { + return { props }; + }, + template: `<Component v-bind="props">${children.join('\n')}</Component>`, + }); + + const app = createSSRApp(App); + const html = await renderToString(app); + return html; + }; + }, + imports: { + vue: ['createApp', 'h: createElement'], + }, + render({ Component, root, props, children }) { + const vueProps = cleanPropsForVue(JSON.parse(props)); + return `const App = { render: () => createElement(${Component}, ${JSON.stringify(vueProps)}, { default: () => ${children} }) }; +createApp(App).mount(${root});`; + }, +}; + +const renderer = createRenderer(Vue); + +export const __vue_static = renderer.static; +export const __vue_load = renderer.load; +export const __vue_idle = renderer.idle; +export const __vue_visible = renderer.visible; diff --git a/packages/astro/src/frontend/runtime/svelte.ts b/packages/astro/src/frontend/runtime/svelte.ts new file mode 100644 index 000000000..78b6af6b6 --- /dev/null +++ b/packages/astro/src/frontend/runtime/svelte.ts @@ -0,0 +1,10 @@ +import SvelteWrapper from '../SvelteWrapper.svelte.client'; +import type { SvelteComponent } from 'svelte'; + +export default (target: Element, component: SvelteComponent, props: any, children: string) => { + new SvelteWrapper({ + target, + props: { __astro_component: component, __astro_children: children, ...props }, + hydrate: true, + }); +}; diff --git a/packages/astro/src/logger.ts b/packages/astro/src/logger.ts new file mode 100644 index 000000000..c42c889f1 --- /dev/null +++ b/packages/astro/src/logger.ts @@ -0,0 +1,143 @@ +import type { CompileError } from 'astro-parser'; +import { bold, blue, red, grey, underline } from 'kleur/colors'; +import { Writable } from 'stream'; +import { format as utilFormat } from 'util'; + +type ConsoleStream = Writable & { + fd: 1 | 2; +}; + +export const defaultLogDestination = new Writable({ + objectMode: true, + write(event: LogMessage, _, callback) { + let dest: ConsoleStream = process.stderr; + if (levels[event.level] < levels['error']) { + dest = process.stdout; + } + let type = event.type; + if(type !== null) { + if (event.level === 'info') { + type = bold(blue(type)); + } else if (event.level === 'error') { + type = bold(red(type)); + } + + dest.write(`[${type}] `); + } + + dest.write(utilFormat(...event.args)); + dest.write('\n'); + + callback(); + }, +}); + +interface LogWritable<T> extends Writable { + write: (chunk: T) => boolean; +} + +export type LoggerLevel = 'debug' | 'info' | 'warn' | 'error' | 'silent'; // same as Pino +export type LoggerEvent = 'debug' | 'info' | 'warn' | 'error'; + +export interface LogOptions { + dest: LogWritable<LogMessage>; + level: LoggerLevel; +} + +export const defaultLogOptions: LogOptions = { + dest: defaultLogDestination, + level: 'info', +}; + +export interface LogMessage { + type: string | null; + level: LoggerLevel; + message: string; + args: Array<any>; +} + +const levels: Record<LoggerLevel, number> = { + debug: 20, + info: 30, + warn: 40, + error: 50, + silent: 90, +}; + +/** Full logging API */ +export function log(opts: LogOptions = defaultLogOptions, level: LoggerLevel, type: string | null, ...args: Array<any>) { + const event: LogMessage = { + type, + level, + args, + message: '', + }; + + // test if this level is enabled or not + if (levels[opts.level] > levels[level]) { + return; // do nothing + } + + opts.dest.write(event); +} + +/** Emit a message only shown in debug mode */ +export function debug(opts: LogOptions, type: string | null, ...messages: Array<any>) { + return log(opts, 'debug', type, ...messages); +} + +/** Emit a general info message (be careful using this too much!) */ +export function info(opts: LogOptions, type: string | null, ...messages: Array<any>) { + return log(opts, 'info', type, ...messages); +} + +/** Emit a warning a user should be aware of */ +export function warn(opts: LogOptions, type: string | null, ...messages: Array<any>) { + return log(opts, 'warn', type, ...messages); +} + +/** Emit a fatal error message the user should address. */ +export function error(opts: LogOptions, type: string | null, ...messages: Array<any>) { + return log(opts, 'error', type, ...messages); +} + +/** Pretty format error for display */ +export function parseError(opts: LogOptions, err: CompileError) { + let frame = err.frame + // Switch colons for pipes + .replace(/^([0-9]+)(:)/gm, `${bold('$1')} │`) + // Make the caret red. + .replace(/(?<=^\s+)(\^)/gm, bold(red(' ^'))) + // Add identation + .replace(/^/gm, ' '); + + error( + opts, + 'parse-error', + ` + + ${underline(bold(grey(`${err.filename}:${err.start.line}:${err.start.column}`)))} + + ${bold(red(`𝘅 ${err.message}`))} + +${frame} +` + ); +} + +// A default logger for when too lazy to pass LogOptions around. +export const logger = { + debug: debug.bind(null, defaultLogOptions), + info: info.bind(null, defaultLogOptions), + warn: warn.bind(null, defaultLogOptions), + error: error.bind(null, defaultLogOptions), +}; + +// For silencing libraries that go directly to console.warn +export function trapWarn(cb: (...args: any[]) => void = () =>{}) { + const warn = console.warn; + console.warn = function(...args: any[]) { + cb(...args); + }; + return () => console.warn = warn; +} diff --git a/packages/astro/src/runtime.ts b/packages/astro/src/runtime.ts new file mode 100644 index 000000000..5369996f4 --- /dev/null +++ b/packages/astro/src/runtime.ts @@ -0,0 +1,365 @@ +import { fileURLToPath } from 'url'; +import type { SnowpackDevServer, ServerRuntime as SnowpackServerRuntime, SnowpackConfig } from 'snowpack'; +import type { AstroConfig, CollectionResult, CollectionRSS, CreateCollection, Params, RuntimeMode } from './@types/astro'; +import type { LogOptions } from './logger'; +import type { CompileError } from 'astro-parser'; +import { debug, info } from './logger.js'; +import { searchForPage } from './search.js'; + +import { existsSync } from 'fs'; +import { loadConfiguration, logger as snowpackLogger, startServer as startSnowpackServer } from 'snowpack'; + +// We need to use require.resolve for snowpack plugins, so create a require function here. +import { createRequire } from 'module'; +const require = createRequire(import.meta.url); + +interface RuntimeConfig { + astroConfig: AstroConfig; + logging: LogOptions; + mode: RuntimeMode; + backendSnowpack: SnowpackDevServer; + backendSnowpackRuntime: SnowpackServerRuntime; + backendSnowpackConfig: SnowpackConfig; + frontendSnowpack: SnowpackDevServer; + frontendSnowpackRuntime: SnowpackServerRuntime; + frontendSnowpackConfig: SnowpackConfig; +} + +// info needed for collection generation +interface CollectionInfo { + additionalURLs: Set<string>; + rss?: { data: any[] & CollectionRSS }; +} + +type LoadResultSuccess = { + statusCode: 200; + contents: string | Buffer; + contentType?: string | false; +}; +type LoadResultNotFound = { statusCode: 404; error: Error; collectionInfo?: CollectionInfo }; +type LoadResultRedirect = { statusCode: 301 | 302; location: string; collectionInfo?: CollectionInfo }; +type LoadResultError = { statusCode: 500 } & ({ type: 'parse-error'; error: CompileError } | { type: 'unknown'; error: Error }); + +export type LoadResult = (LoadResultSuccess | LoadResultNotFound | LoadResultRedirect | LoadResultError) & { collectionInfo?: CollectionInfo }; + +// Disable snowpack from writing to stdout/err. +snowpackLogger.level = 'silent'; + +/** Pass a URL to Astro to resolve and build */ +async function load(config: RuntimeConfig, rawPathname: string | undefined): Promise<LoadResult> { + const { logging, backendSnowpackRuntime, frontendSnowpack } = config; + const { astroRoot } = config.astroConfig; + + const fullurl = new URL(rawPathname || '/', 'https://example.org/'); + + const reqPath = decodeURI(fullurl.pathname); + info(logging, 'access', reqPath); + + const searchResult = searchForPage(fullurl, astroRoot); + if (searchResult.statusCode === 404) { + try { + const result = await frontendSnowpack.loadUrl(reqPath); + if (!result) throw new Error(`Unable to load ${reqPath}`); + // success + return { + statusCode: 200, + ...result, + }; + } catch (err) { + // build error + if (err.failed) { + return { statusCode: 500, type: 'unknown', error: err }; + } + + // not found + return { statusCode: 404, error: err }; + } + } + + if (searchResult.statusCode === 301) { + return { statusCode: 301, location: searchResult.pathname }; + } + + const snowpackURL = searchResult.location.snowpackURL; + let rss: { data: any[] & CollectionRSS } = {} as any; + + try { + const mod = await backendSnowpackRuntime.importModule(snowpackURL); + debug(logging, 'resolve', `${reqPath} -> ${snowpackURL}`); + + // handle collection + let collection = {} as CollectionResult; + let additionalURLs = new Set<string>(); + + if (mod.exports.createCollection) { + const createCollection: CreateCollection = await mod.exports.createCollection(); + for (const key of Object.keys(createCollection)) { + if (key !== 'data' && key !== 'routes' && key !== 'permalink' && key !== 'pageSize' && key !== 'rss') { + throw new Error(`[createCollection] unknown option: "${key}"`); + } + } + let { data: loadData, routes, permalink, pageSize, rss: createRSS } = createCollection; + if (!pageSize) pageSize = 25; // can’t be 0 + let currentParams: Params = {}; + + // params + if (routes || permalink) { + if (!routes || !permalink) { + throw new Error('createCollection() must have both routes and permalink options. Include both together, or omit both.'); + } + let requestedParams = routes.find((p) => { + const baseURL = (permalink as any)({ params: p }); + additionalURLs.add(baseURL); + return baseURL === reqPath || `${baseURL}/${searchResult.currentPage || 1}` === reqPath; + }); + if (requestedParams) { + currentParams = requestedParams; + collection.params = requestedParams; + } + } + + let data: any[] = await loadData({ params: currentParams }); + + // handle RSS + if (createRSS) { + rss = { + ...createRSS, + data: [...data] as any, + }; + } + + collection.start = 0; + collection.end = data.length - 1; + collection.total = data.length; + collection.page = { current: 1, size: pageSize, last: 1 }; + collection.url = { current: reqPath }; + + // paginate + if (searchResult.currentPage) { + const start = (searchResult.currentPage - 1) * pageSize; // currentPage is 1-indexed + const end = Math.min(start + pageSize, data.length); + + collection.start = start; + collection.end = end - 1; + collection.page.current = searchResult.currentPage; + collection.page.last = Math.ceil(data.length / pageSize); + // TODO: fix the .replace() hack + if (end < data.length) { + collection.url.next = collection.url.current.replace(/(\/\d+)?$/, `/${searchResult.currentPage + 1}`); + } + if (searchResult.currentPage > 1) { + collection.url.prev = collection.url.current + .replace(/\d+$/, `${searchResult.currentPage - 1 || 1}`) // update page # + .replace(/\/1$/, ''); // if end is `/1`, then just omit + } + + // from page 2 to the end, add all pages as additional URLs (needed for build) + for (let n = 1; n <= collection.page.last; n++) { + if (additionalURLs.size) { + // if this is a param-based collection, paginate all params + additionalURLs.forEach((url) => { + additionalURLs.add(url.replace(/(\/\d+)?$/, `/${n}`)); + }); + } else { + // if this has no params, simply add page + additionalURLs.add(reqPath.replace(/(\/\d+)?$/, `/${n}`)); + } + } + + data = data.slice(start, end); + } else if (createCollection.pageSize) { + // TODO: fix bug where redirect doesn’t happen + // This happens because a pageSize is set, but the user isn’t on a paginated route. Redirect: + return { + statusCode: 301, + location: reqPath + '/1', + collectionInfo: { + additionalURLs, + rss: rss.data ? rss : undefined, + }, + }; + } + + // if we’ve paginated too far, this is a 404 + if (!data.length) { + return { + statusCode: 404, + error: new Error('Not Found'), + collectionInfo: { + additionalURLs, + rss: rss.data ? rss : undefined, + }, + }; + } + + collection.data = data; + } + + const requestURL = new URL(fullurl.toString()); + + // For first release query params are not passed to components. + // An exception is made for dev server specific routes. + if(reqPath !== '/500') { + requestURL.search = ''; + } + + let html = (await mod.exports.__renderPage({ + request: { + // params should go here when implemented + url: requestURL + }, + children: [], + props: { collection }, + })) as string; + + // inject styles + // TODO: handle this in compiler + const styleTags = Array.isArray(mod.css) && mod.css.length ? mod.css.reduce((markup, href) => `${markup}\n<link rel="stylesheet" type="text/css" href="${href}" />`, '') : ``; + if (html.indexOf('</head>') !== -1) { + html = html.replace('</head>', `${styleTags}</head>`); + } else { + html = styleTags + html; + } + + return { + statusCode: 200, + contentType: 'text/html; charset=utf-8', + contents: html, + collectionInfo: { + additionalURLs, + rss: rss.data ? rss : undefined, + }, + }; + } catch (err) { + if (err.code === 'parse-error' || err instanceof SyntaxError) { + return { + statusCode: 500, + type: 'parse-error', + error: err, + }; + } + return { + statusCode: 500, + type: 'unknown', + error: err, + }; + } +} + +export interface AstroRuntime { + runtimeConfig: RuntimeConfig; + load: (rawPathname: string | undefined) => Promise<LoadResult>; + shutdown: () => Promise<void>; +} + +interface RuntimeOptions { + mode: RuntimeMode; + logging: LogOptions; +} + +interface CreateSnowpackOptions { + env: Record<string, any>; + mode: RuntimeMode; + resolvePackageUrl?: (pkgName: string) => Promise<string>; +} + +/** Create a new Snowpack instance to power Astro */ +async function createSnowpack(astroConfig: AstroConfig, options: CreateSnowpackOptions) { + const { projectRoot, astroRoot, extensions } = astroConfig; + const { env, mode, resolvePackageUrl } = options; + + const internalPath = new URL('./frontend/', import.meta.url); + + let snowpack: SnowpackDevServer; + const astroPlugOptions: { + resolvePackageUrl?: (s: string) => Promise<string>; + extensions?: Record<string, string>; + astroConfig: AstroConfig; + } = { + astroConfig, + extensions, + resolvePackageUrl, + }; + + const mountOptions = { + [fileURLToPath(astroRoot)]: '/_astro', + [fileURLToPath(internalPath)]: '/_astro_internal', + }; + + if (existsSync(astroConfig.public)) { + mountOptions[fileURLToPath(astroConfig.public)] = '/'; + } + + const snowpackConfig = await loadConfiguration({ + root: fileURLToPath(projectRoot), + mount: mountOptions, + mode, + plugins: [ + [fileURLToPath(new URL('../snowpack-plugin.cjs', import.meta.url)), astroPlugOptions], + require.resolve('@snowpack/plugin-sass'), + [require.resolve('@snowpack/plugin-svelte'), { compilerOptions: { hydratable: true } }], + require.resolve('@snowpack/plugin-vue'), + ], + devOptions: { + open: 'none', + output: 'stream', + port: 0, + }, + buildOptions: { + out: astroConfig.dist, + }, + packageOptions: { + knownEntrypoints: ['preact-render-to-string'], + external: ['@vue/server-renderer', 'node-fetch', 'prismjs/components/index.js'], + }, + }); + + const envConfig = snowpackConfig.env || (snowpackConfig.env = {}); + Object.assign(envConfig, env); + + snowpack = await startSnowpackServer({ + config: snowpackConfig, + lockfile: null, + }); + const snowpackRuntime = snowpack.getServerRuntime(); + + return { snowpack, snowpackRuntime, snowpackConfig }; +} + +/** Core Astro runtime */ +export async function createRuntime(astroConfig: AstroConfig, { mode, logging }: RuntimeOptions): Promise<AstroRuntime> { + const resolvePackageUrl = async (pkgName: string) => frontendSnowpack.getUrlForPackage(pkgName); + + const { snowpack: backendSnowpack, snowpackRuntime: backendSnowpackRuntime, snowpackConfig: backendSnowpackConfig } = await createSnowpack(astroConfig, { + env: { + astro: true, + }, + mode, + resolvePackageUrl, + }); + + const { snowpack: frontendSnowpack, snowpackRuntime: frontendSnowpackRuntime, snowpackConfig: frontendSnowpackConfig } = await createSnowpack(astroConfig, { + env: { + astro: false, + }, + mode, + }); + + const runtimeConfig: RuntimeConfig = { + astroConfig, + logging, + mode, + backendSnowpack, + backendSnowpackRuntime, + backendSnowpackConfig, + frontendSnowpack, + frontendSnowpackRuntime, + frontendSnowpackConfig, + }; + + return { + runtimeConfig, + load: load.bind(null, runtimeConfig), + shutdown: () => Promise.all([backendSnowpack.shutdown(), frontendSnowpack.shutdown()]).then(() => void 0), + }; +} diff --git a/packages/astro/src/search.ts b/packages/astro/src/search.ts new file mode 100644 index 000000000..c141e4a77 --- /dev/null +++ b/packages/astro/src/search.ts @@ -0,0 +1,141 @@ +import { existsSync } from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { fdir, PathsOutput } from 'fdir'; + +interface PageLocation { + fileURL: URL; + snowpackURL: string; +} +/** findAnyPage and return the _astro candidate for snowpack */ +function findAnyPage(candidates: Array<string>, astroRoot: URL): PageLocation | false { + for (let candidate of candidates) { + const url = new URL(`./pages/${candidate}`, astroRoot); + if (existsSync(url)) { + return { + fileURL: url, + snowpackURL: `/_astro/pages/${candidate}.js`, + }; + } + } + return false; +} + +type SearchResult = + | { + statusCode: 200; + location: PageLocation; + pathname: string; + currentPage?: number; + } + | { + statusCode: 301; + location: null; + pathname: string; + } + | { + statusCode: 404; + }; + +/** Given a URL, attempt to locate its source file (similar to Snowpack’s load()) */ +export function searchForPage(url: URL, astroRoot: URL): SearchResult { + const reqPath = decodeURI(url.pathname); + const base = reqPath.substr(1); + + // Try to find index.astro/md paths + if (reqPath.endsWith('/')) { + const candidates = [`${base}index.astro`, `${base}index.md`]; + const location = findAnyPage(candidates, astroRoot); + if (location) { + return { + statusCode: 200, + location, + pathname: reqPath, + }; + } + } else { + // Try to find the page by its name. + const candidates = [`${base}.astro`, `${base}.md`]; + let location = findAnyPage(candidates, astroRoot); + if (location) { + return { + statusCode: 200, + location, + pathname: reqPath, + }; + } + } + + // Try to find name/index.astro/md + const candidates = [`${base}/index.astro`, `${base}/index.md`]; + const location = findAnyPage(candidates, astroRoot); + if (location) { + return { + statusCode: 301, + location: null, + pathname: reqPath + '/', + }; + } + + // Try and load collections (but only for non-extension files) + const hasExt = !!path.extname(reqPath); + if (!location && !hasExt) { + const collection = loadCollection(reqPath, astroRoot); + if (collection) { + return { + statusCode: 200, + location: collection.location, + pathname: reqPath, + currentPage: collection.currentPage || 1, + }; + } + } + + if(reqPath === '/500') { + return { + statusCode: 200, + location: { + fileURL: new URL('./frontend/500.astro', import.meta.url), + snowpackURL: `/_astro_internal/500.astro.js` + }, + pathname: reqPath + }; + } + + return { + statusCode: 404, + }; +} + +const crawler = new fdir(); + +/** load a collection route */ +function loadCollection(url: string, astroRoot: URL): { currentPage?: number; location: PageLocation } | undefined { + const pages = (crawler + .glob('**/*') + .crawl(path.join(fileURLToPath(astroRoot), 'pages')) + .sync() as PathsOutput).filter((filepath) => filepath.startsWith('$') || filepath.includes('/$')); + for (const pageURL of pages) { + const reqURL = new RegExp('^/' + pageURL.replace(/\$([^/]+)\.astro/, '$1') + '/?(.*)'); + const match = url.match(reqURL); + if (match) { + let currentPage: number | undefined; + if (match[1]) { + const segments = match[1].split('/').filter((s) => !!s); + if (segments.length) { + const last = segments.pop() as string; + if (parseInt(last, 10)) { + currentPage = parseInt(last, 10); + } + } + } + return { + location: { + fileURL: new URL(`./pages/${pageURL}`, astroRoot), + snowpackURL: `/_astro/pages/${pageURL}.js`, + }, + currentPage, + }; + } + } +} |