From 6a7cf0712da23e2c095f4bc4f2512e618bceb38e Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Mon, 13 Mar 2023 15:36:11 -0400 Subject: Prevent astro:content from depending on Node builtins (#6537) * Prevent astro:content from depending on Node builtins * Right file * Move the plugin into test-plugins.js --- .changeset/four-planets-smoke.md | 5 + packages/astro/package.json | 2 + packages/astro/src/content/internal.ts | 227 --------------------- packages/astro/src/content/runtime-assets.ts | 28 +++ packages/astro/src/content/runtime.ts | 199 ++++++++++++++++++ .../src/content/template/virtual-mod-assets.mjs | 9 + .../astro/src/content/template/virtual-mod.mjs | 8 +- packages/astro/src/content/utils.ts | 2 + .../src/content/vite-plugin-content-virtual-mod.ts | 9 +- packages/astro/test/content-collections.test.js | 6 + packages/astro/test/test-plugins.js | 17 ++ 11 files changed, 276 insertions(+), 236 deletions(-) create mode 100644 .changeset/four-planets-smoke.md delete mode 100644 packages/astro/src/content/internal.ts create mode 100644 packages/astro/src/content/runtime-assets.ts create mode 100644 packages/astro/src/content/runtime.ts create mode 100644 packages/astro/src/content/template/virtual-mod-assets.mjs create mode 100644 packages/astro/test/test-plugins.js diff --git a/.changeset/four-planets-smoke.md b/.changeset/four-planets-smoke.md new file mode 100644 index 000000000..7c667aa57 --- /dev/null +++ b/.changeset/four-planets-smoke.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Prevent astro:content from depending on Node builtins diff --git a/packages/astro/package.json b/packages/astro/package.json index 23410941c..274694d8e 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -53,6 +53,8 @@ "./assets/services/sharp": "./dist/assets/services/sharp.js", "./assets/services/squoosh": "./dist/assets/services/squoosh.js", "./content/internal": "./dist/content/internal.js", + "./content/runtime": "./dist/content/runtime.js", + "./content/runtime-assets": "./dist/content/runtime-assets.js", "./debug": "./components/Debug.astro", "./internal/*": "./dist/runtime/server/*", "./package.json": "./package.json", diff --git a/packages/astro/src/content/internal.ts b/packages/astro/src/content/internal.ts deleted file mode 100644 index 951a52f6d..000000000 --- a/packages/astro/src/content/internal.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { z } from 'zod'; -import { imageMetadata, type Metadata } from '../assets/utils/metadata.js'; -import { AstroError, AstroErrorData } from '../core/errors/index.js'; -import { prependForwardSlash } from '../core/path.js'; - -import { - createComponent, - createHeadAndContent, - createScopedResult, - renderComponent, - renderScriptElement, - renderStyleElement, - renderTemplate, - renderUniqueStylesheet, - unescapeHTML, -} from '../runtime/server/index.js'; - -type GlobResult = Record Promise>; -type CollectionToEntryMap = Record; - -export function createCollectionToGlobResultMap({ - globResult, - contentDir, -}: { - globResult: GlobResult; - contentDir: string; -}) { - const collectionToGlobResultMap: CollectionToEntryMap = {}; - for (const key in globResult) { - const keyRelativeToContentDir = key.replace(new RegExp(`^${contentDir}`), ''); - const segments = keyRelativeToContentDir.split('/'); - if (segments.length <= 1) continue; - const collection = segments[0]; - const entryId = segments.slice(1).join('/'); - collectionToGlobResultMap[collection] ??= {}; - collectionToGlobResultMap[collection][entryId] = globResult[key]; - } - return collectionToGlobResultMap; -} - -const cacheEntriesByCollection = new Map(); -export function createGetCollection({ - collectionToEntryMap, - collectionToRenderEntryMap, -}: { - collectionToEntryMap: CollectionToEntryMap; - collectionToRenderEntryMap: CollectionToEntryMap; -}) { - return async function getCollection(collection: string, filter?: (entry: any) => unknown) { - const lazyImports = Object.values(collectionToEntryMap[collection] ?? {}); - let entries: any[] = []; - // Cache `getCollection()` calls in production only - // prevents stale cache in development - if (import.meta.env.PROD && cacheEntriesByCollection.has(collection)) { - entries = cacheEntriesByCollection.get(collection)!; - } else { - entries = await Promise.all( - lazyImports.map(async (lazyImport) => { - const entry = await lazyImport(); - return { - id: entry.id, - slug: entry.slug, - body: entry.body, - collection: entry.collection, - data: entry.data, - async render() { - return render({ - collection: entry.collection, - id: entry.id, - collectionToRenderEntryMap, - }); - }, - }; - }) - ); - cacheEntriesByCollection.set(collection, entries); - } - if (typeof filter === 'function') { - return entries.filter(filter); - } else { - return entries; - } - }; -} - -export function createGetEntryBySlug({ - getCollection, - collectionToRenderEntryMap, -}: { - getCollection: ReturnType; - collectionToRenderEntryMap: CollectionToEntryMap; -}) { - return async function getEntryBySlug(collection: string, slug: string) { - // This is not an optimized lookup. Should look into an O(1) implementation - // as it's probably that people will have very large collections. - const entries = await getCollection(collection); - let candidate: (typeof entries)[number] | undefined = undefined; - for (let entry of entries) { - if (entry.slug === slug) { - candidate = entry; - break; - } - } - - if (typeof candidate === 'undefined') { - return undefined; - } - - const entry = candidate; - return { - id: entry.id, - slug: entry.slug, - body: entry.body, - collection: entry.collection, - data: entry.data, - async render() { - return render({ - collection: entry.collection, - id: entry.id, - collectionToRenderEntryMap, - }); - }, - }; - }; -} - -async function render({ - collection, - id, - collectionToRenderEntryMap, -}: { - collection: string; - id: string; - collectionToRenderEntryMap: CollectionToEntryMap; -}) { - const UnexpectedRenderError = new AstroError({ - ...AstroErrorData.UnknownContentCollectionError, - message: `Unexpected error while rendering ${String(collection)} → ${String(id)}.`, - }); - - const lazyImport = collectionToRenderEntryMap[collection]?.[id]; - if (typeof lazyImport !== 'function') throw UnexpectedRenderError; - - const baseMod = await lazyImport(); - if (baseMod == null || typeof baseMod !== 'object') throw UnexpectedRenderError; - - const { collectedStyles, collectedLinks, collectedScripts, getMod } = baseMod; - if (typeof getMod !== 'function') throw UnexpectedRenderError; - const mod = await getMod(); - if (mod == null || typeof mod !== 'object') throw UnexpectedRenderError; - - const Content = createComponent({ - factory(result, baseProps, slots) { - let styles = '', - links = '', - scripts = ''; - if (Array.isArray(collectedStyles)) { - styles = collectedStyles.map((style: any) => renderStyleElement(style)).join(''); - } - if (Array.isArray(collectedLinks)) { - links = collectedLinks - .map((link: any) => { - return renderUniqueStylesheet(result, { - href: prependForwardSlash(link), - }); - }) - .join(''); - } - if (Array.isArray(collectedScripts)) { - scripts = collectedScripts.map((script: any) => renderScriptElement(script)).join(''); - } - - let props = baseProps; - // Auto-apply MDX components export - if (id.endsWith('mdx')) { - props = { - components: mod.components ?? {}, - ...baseProps, - }; - } - - return createHeadAndContent( - unescapeHTML(styles + links + scripts) as any, - renderTemplate`${renderComponent( - createScopedResult(result), - 'Content', - mod.Content, - props, - slots - )}` - ); - }, - propagation: 'self', - }); - - return { - Content, - headings: mod.getHeadings?.() ?? [], - remarkPluginFrontmatter: mod.frontmatter ?? {}, - }; -} - -export function createImage(options: { assetsDir: string; relAssetsDir: string }) { - return () => { - if (options.assetsDir === 'undefined') { - throw new Error('Enable `experimental.assets` in your Astro config to use image()'); - } - - return z.string().transform(async (imagePath) => { - const fullPath = new URL(imagePath, options.assetsDir); - return await getImageMetadata(fullPath); - }); - }; -} - -async function getImageMetadata( - imagePath: URL -): Promise<(Metadata & { __astro_asset: true }) | undefined> { - const meta = await imageMetadata(imagePath); - - if (!meta) { - return undefined; - } - - delete meta.orientation; - return { ...meta, __astro_asset: true }; -} diff --git a/packages/astro/src/content/runtime-assets.ts b/packages/astro/src/content/runtime-assets.ts new file mode 100644 index 000000000..f6448b10c --- /dev/null +++ b/packages/astro/src/content/runtime-assets.ts @@ -0,0 +1,28 @@ +import { z } from 'zod'; +import { imageMetadata, type Metadata } from '../assets/utils/metadata.js'; + +export function createImage(options: { assetsDir: string; relAssetsDir: string }) { + return () => { + if (options.assetsDir === 'undefined') { + throw new Error('Enable `experimental.assets` in your Astro config to use image()'); + } + + return z.string().transform(async (imagePath) => { + const fullPath = new URL(imagePath, options.assetsDir); + return await getImageMetadata(fullPath); + }); + }; +} + +async function getImageMetadata( + imagePath: URL +): Promise<(Metadata & { __astro_asset: true }) | undefined> { + const meta = await imageMetadata(imagePath); + + if (!meta) { + return undefined; + } + + delete meta.orientation; + return { ...meta, __astro_asset: true }; +} diff --git a/packages/astro/src/content/runtime.ts b/packages/astro/src/content/runtime.ts new file mode 100644 index 000000000..d58e4be62 --- /dev/null +++ b/packages/astro/src/content/runtime.ts @@ -0,0 +1,199 @@ +import { AstroError, AstroErrorData } from '../core/errors/index.js'; +import { prependForwardSlash } from '../core/path.js'; + +import { + createComponent, + createHeadAndContent, + createScopedResult, + renderComponent, + renderScriptElement, + renderStyleElement, + renderTemplate, + renderUniqueStylesheet, + unescapeHTML, +} from '../runtime/server/index.js'; + +type GlobResult = Record Promise>; +type CollectionToEntryMap = Record; + +export function createCollectionToGlobResultMap({ + globResult, + contentDir, +}: { + globResult: GlobResult; + contentDir: string; +}) { + const collectionToGlobResultMap: CollectionToEntryMap = {}; + for (const key in globResult) { + const keyRelativeToContentDir = key.replace(new RegExp(`^${contentDir}`), ''); + const segments = keyRelativeToContentDir.split('/'); + if (segments.length <= 1) continue; + const collection = segments[0]; + const entryId = segments.slice(1).join('/'); + collectionToGlobResultMap[collection] ??= {}; + collectionToGlobResultMap[collection][entryId] = globResult[key]; + } + return collectionToGlobResultMap; +} + +const cacheEntriesByCollection = new Map(); +export function createGetCollection({ + collectionToEntryMap, + collectionToRenderEntryMap, +}: { + collectionToEntryMap: CollectionToEntryMap; + collectionToRenderEntryMap: CollectionToEntryMap; +}) { + return async function getCollection(collection: string, filter?: (entry: any) => unknown) { + const lazyImports = Object.values(collectionToEntryMap[collection] ?? {}); + let entries: any[] = []; + // Cache `getCollection()` calls in production only + // prevents stale cache in development + if (import.meta.env.PROD && cacheEntriesByCollection.has(collection)) { + entries = cacheEntriesByCollection.get(collection)!; + } else { + entries = await Promise.all( + lazyImports.map(async (lazyImport) => { + const entry = await lazyImport(); + return { + id: entry.id, + slug: entry.slug, + body: entry.body, + collection: entry.collection, + data: entry.data, + async render() { + return render({ + collection: entry.collection, + id: entry.id, + collectionToRenderEntryMap, + }); + }, + }; + }) + ); + cacheEntriesByCollection.set(collection, entries); + } + if (typeof filter === 'function') { + return entries.filter(filter); + } else { + return entries; + } + }; +} + +export function createGetEntryBySlug({ + getCollection, + collectionToRenderEntryMap, +}: { + getCollection: ReturnType; + collectionToRenderEntryMap: CollectionToEntryMap; +}) { + return async function getEntryBySlug(collection: string, slug: string) { + // This is not an optimized lookup. Should look into an O(1) implementation + // as it's probably that people will have very large collections. + const entries = await getCollection(collection); + let candidate: (typeof entries)[number] | undefined = undefined; + for (let entry of entries) { + if (entry.slug === slug) { + candidate = entry; + break; + } + } + + if (typeof candidate === 'undefined') { + return undefined; + } + + const entry = candidate; + return { + id: entry.id, + slug: entry.slug, + body: entry.body, + collection: entry.collection, + data: entry.data, + async render() { + return render({ + collection: entry.collection, + id: entry.id, + collectionToRenderEntryMap, + }); + }, + }; + }; +} + +async function render({ + collection, + id, + collectionToRenderEntryMap, +}: { + collection: string; + id: string; + collectionToRenderEntryMap: CollectionToEntryMap; +}) { + const UnexpectedRenderError = new AstroError({ + ...AstroErrorData.UnknownContentCollectionError, + message: `Unexpected error while rendering ${String(collection)} → ${String(id)}.`, + }); + + const lazyImport = collectionToRenderEntryMap[collection]?.[id]; + if (typeof lazyImport !== 'function') throw UnexpectedRenderError; + + const baseMod = await lazyImport(); + if (baseMod == null || typeof baseMod !== 'object') throw UnexpectedRenderError; + + const { collectedStyles, collectedLinks, collectedScripts, getMod } = baseMod; + if (typeof getMod !== 'function') throw UnexpectedRenderError; + const mod = await getMod(); + if (mod == null || typeof mod !== 'object') throw UnexpectedRenderError; + + const Content = createComponent({ + factory(result, baseProps, slots) { + let styles = '', + links = '', + scripts = ''; + if (Array.isArray(collectedStyles)) { + styles = collectedStyles.map((style: any) => renderStyleElement(style)).join(''); + } + if (Array.isArray(collectedLinks)) { + links = collectedLinks + .map((link: any) => { + return renderUniqueStylesheet(result, { + href: prependForwardSlash(link), + }); + }) + .join(''); + } + if (Array.isArray(collectedScripts)) { + scripts = collectedScripts.map((script: any) => renderScriptElement(script)).join(''); + } + + let props = baseProps; + // Auto-apply MDX components export + if (id.endsWith('mdx')) { + props = { + components: mod.components ?? {}, + ...baseProps, + }; + } + + return createHeadAndContent( + unescapeHTML(styles + links + scripts) as any, + renderTemplate`${renderComponent( + createScopedResult(result), + 'Content', + mod.Content, + props, + slots + )}` + ); + }, + propagation: 'self', + }); + + return { + Content, + headings: mod.getHeadings?.() ?? [], + remarkPluginFrontmatter: mod.frontmatter ?? {}, + }; +} diff --git a/packages/astro/src/content/template/virtual-mod-assets.mjs b/packages/astro/src/content/template/virtual-mod-assets.mjs new file mode 100644 index 000000000..32b41dd97 --- /dev/null +++ b/packages/astro/src/content/template/virtual-mod-assets.mjs @@ -0,0 +1,9 @@ +import { + createImage +} from 'astro/content/runtime-assets'; + +const assetsDir = '@@ASSETS_DIR@@'; + +export const image = createImage({ + assetsDir, +}); diff --git a/packages/astro/src/content/template/virtual-mod.mjs b/packages/astro/src/content/template/virtual-mod.mjs index 5ce29dcf8..c5dc1b4f3 100644 --- a/packages/astro/src/content/template/virtual-mod.mjs +++ b/packages/astro/src/content/template/virtual-mod.mjs @@ -3,8 +3,7 @@ import { createCollectionToGlobResultMap, createGetCollection, createGetEntryBySlug, - createImage, -} from 'astro/content/internal'; +} from 'astro/content/runtime'; export { z } from 'astro/zod'; @@ -13,7 +12,6 @@ export function defineCollection(config) { } const contentDir = '@@CONTENT_DIR@@'; -const assetsDir = '@@ASSETS_DIR@@'; const entryGlob = import.meta.glob('@@ENTRY_GLOB_PATH@@', { query: { astroContent: true }, @@ -40,7 +38,3 @@ export const getEntryBySlug = createGetEntryBySlug({ getCollection, collectionToRenderEntryMap, }); - -export const image = createImage({ - assetsDir, -}); diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index 2db053dd3..f6b420acd 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -334,6 +334,7 @@ export type ContentPaths = { cacheDir: URL; typesTemplate: URL; virtualModTemplate: URL; + virtualAssetsModTemplate: URL; config: { exists: boolean; url: URL; @@ -352,6 +353,7 @@ export function getContentPaths( assetsDir: new URL('./assets/', srcDir), typesTemplate: new URL('types.d.ts', templateDir), virtualModTemplate: new URL('virtual-mod.mjs', templateDir), + virtualAssetsModTemplate: new URL('virtual-mod-assets.mjs', templateDir), config: configStats, }; } diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index 99b6e3f3c..6691326e9 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -36,11 +36,16 @@ export function astroContentVirtualModPlugin({ const virtualModContents = fsMod .readFileSync(contentPaths.virtualModTemplate, 'utf-8') .replace('@@CONTENT_DIR@@', relContentDir) - .replace('@@ASSETS_DIR@@', assetsDir) .replace('@@ENTRY_GLOB_PATH@@', entryGlob) .replace('@@RENDER_ENTRY_GLOB_PATH@@', entryGlob); + const virtualAssetsModContents = fsMod + .readFileSync(contentPaths.virtualAssetsModTemplate, 'utf-8') + .replace('@@ASSETS_DIR@@', assetsDir); const astroContentVirtualModuleId = '\0' + VIRTUAL_MODULE_ID; + const allContents = settings.config.experimental.assets ? + (virtualModContents + virtualAssetsModContents) : + virtualModContents; return { name: 'astro-content-virtual-mod-plugin', @@ -53,7 +58,7 @@ export function astroContentVirtualModPlugin({ load(id) { if (id === astroContentVirtualModuleId) { return { - code: virtualModContents, + code: allContents, }; } }, diff --git a/packages/astro/test/content-collections.test.js b/packages/astro/test/content-collections.test.js index dd86288d1..6ede7cfc1 100644 --- a/packages/astro/test/content-collections.test.js +++ b/packages/astro/test/content-collections.test.js @@ -3,6 +3,7 @@ import * as cheerio from 'cheerio'; import { expect } from 'chai'; import { loadFixture } from './test-utils.js'; import testAdapter from './test-adapter.js'; +import { preventNodeBuiltinDependencyPlugin } from './test-plugins.js'; describe('Content Collections', () => { describe('Query', () => { @@ -222,6 +223,11 @@ describe('Content Collections', () => { root: './fixtures/content-ssr-integration/', output: 'server', adapter: testAdapter(), + vite: { + plugins: [ + preventNodeBuiltinDependencyPlugin() + ] + } }); await fixture.build(); app = await fixture.loadTestAdapterApp(); diff --git a/packages/astro/test/test-plugins.js b/packages/astro/test/test-plugins.js new file mode 100644 index 000000000..75f944789 --- /dev/null +++ b/packages/astro/test/test-plugins.js @@ -0,0 +1,17 @@ + +export function preventNodeBuiltinDependencyPlugin() { + // Verifies that `astro:content` does not have a hard dependency on Node builtins. + // This is to verify it will run on Cloudflare and Deno + return { + name: 'verify-no-node-stuff', + generateBundle() { + const nodeModules = ['node:fs', 'node:url', 'node:worker_threads', 'node:path']; + nodeModules.forEach(name => { + const mod = this.getModuleInfo(name); + if(mod) { + throw new Error(`Node builtins snuck in: ${name}`) + } + }); + } + }; +} -- cgit v1.2.3