diff options
31 files changed, 1258 insertions, 146 deletions
diff --git a/.changeset/pretty-doodles-wash.md b/.changeset/pretty-doodles-wash.md new file mode 100644 index 000000000..4b27a0efd --- /dev/null +++ b/.changeset/pretty-doodles-wash.md @@ -0,0 +1,5 @@ +--- +'astro': minor +--- + +Adds support for live content collections diff --git a/examples/with-markdoc/src/content.config.ts b/examples/with-markdoc/src/content.config.ts index 79743326e..a991e1ea1 100644 --- a/examples/with-markdoc/src/content.config.ts +++ b/examples/with-markdoc/src/content.config.ts @@ -1,5 +1,5 @@ import { defineCollection } from 'astro:content'; export const collections = { - docs: defineCollection({}) + docs: defineCollection({}), }; diff --git a/packages/astro/src/config/content.ts b/packages/astro/src/config/content.ts new file mode 100644 index 000000000..c486988cb --- /dev/null +++ b/packages/astro/src/config/content.ts @@ -0,0 +1,172 @@ +import type { ZodLiteral, ZodNumber, ZodObject, ZodString, ZodType, ZodUnion } from 'zod'; +import { CONTENT_LAYER_TYPE, LIVE_CONTENT_TYPE } from '../content/consts.js'; +import type { LiveLoader, Loader } from '../content/loaders/types.js'; +import { AstroError, AstroErrorData, AstroUserError } from '../core/errors/index.js'; + +function getImporterFilename() { + // Find the first line in the stack trace that doesn't include 'defineCollection' or 'getImporterFilename' + const stackLine = new Error().stack + ?.split('\n') + .find( + (line) => + !line.includes('defineCollection') && + !line.includes('getImporterFilename') && + line !== 'Error', + ); + if (!stackLine) { + return undefined; + } + // Extract the relative path from the stack line + const match = /\/((?:src|chunks)\/.*?):\d+:\d+/.exec(stackLine); + + return match?.[1] ?? undefined; +} + +// This needs to be in sync with ImageMetadata +export type ImageFunction = () => ZodObject<{ + src: ZodString; + width: ZodNumber; + height: ZodNumber; + format: ZodUnion< + [ + ZodLiteral<'png'>, + ZodLiteral<'jpg'>, + ZodLiteral<'jpeg'>, + ZodLiteral<'tiff'>, + ZodLiteral<'webp'>, + ZodLiteral<'gif'>, + ZodLiteral<'svg'>, + ZodLiteral<'avif'>, + ] + >; +}>; + +export interface DataEntry { + id: string; + data: Record<string, unknown>; + filePath?: string; + body?: string; +} + +export interface DataStore { + get: (key: string) => DataEntry; + entries: () => Array<[id: string, DataEntry]>; + set: (key: string, data: Record<string, unknown>, body?: string, filePath?: string) => void; + values: () => Array<DataEntry>; + keys: () => Array<string>; + delete: (key: string) => void; + clear: () => void; + has: (key: string) => boolean; +} + +export interface MetaStore { + get: (key: string) => string | undefined; + set: (key: string, value: string) => void; + delete: (key: string) => void; + has: (key: string) => boolean; +} + +export type BaseSchema = ZodType; + +export type SchemaContext = { image: ImageFunction }; + +type ContentLayerConfig<S extends BaseSchema, TData extends { id: string } = { id: string }> = { + type?: 'content_layer'; + schema?: S | ((context: SchemaContext) => S); + loader: + | Loader + | (() => + | Array<TData> + | Promise<Array<TData>> + | Record<string, Omit<TData, 'id'> & { id?: string }> + | Promise<Record<string, Omit<TData, 'id'> & { id?: string }>>); +}; + +type DataCollectionConfig<S extends BaseSchema> = { + type: 'data'; + schema?: S | ((context: SchemaContext) => S); +}; + +type ContentCollectionConfig<S extends BaseSchema> = { + type?: 'content'; + schema?: S | ((context: SchemaContext) => S); + loader?: never; +}; + +type LiveDataCollectionConfig<S extends BaseSchema, L extends LiveLoader> = { + type: 'live'; + schema?: S; + loader: L; +}; + +export type BaseCollectionConfig<S extends BaseSchema> = + | ContentCollectionConfig<S> + | DataCollectionConfig<S> + | ContentLayerConfig<S>; + +export type CollectionConfig< + S extends BaseSchema, + TLiveLoader = never, +> = TLiveLoader extends LiveLoader + ? LiveDataCollectionConfig<S, TLiveLoader> + : BaseCollectionConfig<S>; + +export function defineCollection<S extends BaseSchema, TLiveLoader = undefined>( + config: CollectionConfig<S, TLiveLoader>, +): CollectionConfig<S, TLiveLoader> { + const importerFilename = getImporterFilename(); + const isInLiveConfig = importerFilename?.includes('live.config'); + + if (config.type === LIVE_CONTENT_TYPE) { + if (!isInLiveConfig) { + throw new AstroError({ + ...AstroErrorData.LiveContentConfigError, + message: AstroErrorData.LiveContentConfigError.message( + 'Collections with type `live` must be defined in a `src/live.config.ts` file.', + importerFilename ?? 'your content config file', + ), + }); + } + if (!config.loader) { + throw new AstroError({ + ...AstroErrorData.LiveContentConfigError, + message: AstroErrorData.LiveContentConfigError.message( + 'Collections with type `live` must have a `loader` defined.', + importerFilename, + ), + }); + } + if (config.schema) { + if (typeof config.schema === 'function') { + throw new AstroError({ + ...AstroErrorData.LiveContentConfigError, + message: AstroErrorData.LiveContentConfigError.message( + 'The schema cannot be a function for live collections. Please use a schema object instead.', + importerFilename, + ), + }); + } + } + return config; + } + if (isInLiveConfig) { + throw new AstroError({ + ...AstroErrorData.LiveContentConfigError, + message: AstroErrorData.LiveContentConfigError.message( + 'Collections in a `live.config.ts` file must have a type of `live`.', + getImporterFilename(), + ), + }); + } + + if ('loader' in config) { + if (config.type && config.type !== CONTENT_LAYER_TYPE) { + throw new AstroUserError( + `Collections that use the Content Layer API must have a \`loader\` defined and no \`type\` set. Check your collection definitions in ${getImporterFilename() ?? 'your content config file'}.`, + ); + } + config.type = CONTENT_LAYER_TYPE; + } + if (!config.type) config.type = 'content'; + return config; +} diff --git a/packages/astro/src/config/entrypoint.ts b/packages/astro/src/config/entrypoint.ts index dc2f47242..b67d0d862 100644 --- a/packages/astro/src/config/entrypoint.ts +++ b/packages/astro/src/config/entrypoint.ts @@ -2,7 +2,17 @@ import type { SharpImageServiceConfig } from '../assets/services/sharp.js'; import type { ImageServiceConfig } from '../types/public/index.js'; - +export { + defineCollection, + type ImageFunction, + type DataEntry, + type DataStore, + type MetaStore, + type BaseSchema, + type SchemaContext, + type CollectionConfig, + type BaseCollectionConfig, +} from './content.js'; export { defineConfig, getViteConfig } from './index.js'; export { envField } from '../env/config.js'; export { mergeConfig } from '../core/config/merge.js'; diff --git a/packages/astro/src/content/consts.ts b/packages/astro/src/content/consts.ts index 76218d7e8..c2902cb70 100644 --- a/packages/astro/src/content/consts.ts +++ b/packages/astro/src/content/consts.ts @@ -41,3 +41,4 @@ export const COLLECTIONS_MANIFEST_FILE = 'collections/collections.json'; export const COLLECTIONS_DIR = 'collections/'; export const CONTENT_LAYER_TYPE = 'content_layer'; +export const LIVE_CONTENT_TYPE = 'live'; diff --git a/packages/astro/src/content/loaders/errors.ts b/packages/astro/src/content/loaders/errors.ts new file mode 100644 index 000000000..aaa16ac8f --- /dev/null +++ b/packages/astro/src/content/loaders/errors.ts @@ -0,0 +1,62 @@ +import type { ZodError } from "zod"; + +export class LiveCollectionError extends Error { + constructor( + public readonly collection: string, + public readonly message: string, + public readonly cause?: Error, + ) { + super(message); + this.name = 'LiveCollectionError'; + } + static is(error: unknown): error is LiveCollectionError { + return error instanceof LiveCollectionError; + } +} + +export class LiveEntryNotFoundError extends LiveCollectionError { + constructor(collection: string, entryFilter: string | Record<string, unknown>) { + super( + collection, + `Entry ${collection} → ${typeof entryFilter === 'string' ? entryFilter : JSON.stringify(entryFilter)} was not found.`, + ); + this.name = 'LiveEntryNotFoundError'; + } + static is(error: unknown): error is LiveEntryNotFoundError { + return (error as any)?.name === 'LiveEntryNotFoundError'; + } +} + +export class LiveCollectionValidationError extends LiveCollectionError { + constructor(collection: string, entryId: string, error: ZodError) { + super( + collection, + [ + `**${collection} → ${entryId}** data does not match the collection schema.\n`, + ...error.errors.map((zodError) => ` **${zodError.path.join('.')}**: ${zodError.message}`), + '', + ].join('\n'), + ); + this.name = 'LiveCollectionValidationError'; + } + static is(error: unknown): error is LiveCollectionValidationError { + return (error as any)?.name === 'LiveCollectionValidationError'; + } +} + +export class LiveCollectionCacheHintError extends LiveCollectionError { + constructor(collection: string, entryId: string | undefined, error: ZodError) { + super( + collection, + [ + `**${String(collection)}${entryId ? ` → ${String(entryId)}` : ''}** returned an invalid cache hint.\n`, + ...error.errors.map((zodError) => ` **${zodError.path.join('.')}**: ${zodError.message}`), + '', + ].join('\n'), + ); + this.name = 'LiveCollectionCacheHintError'; + } + static is(error: unknown): error is LiveCollectionCacheHintError { + return (error as any)?.name === 'LiveCollectionCacheHintError'; + } +} diff --git a/packages/astro/src/content/loaders/types.ts b/packages/astro/src/content/loaders/types.ts index d41017ead..6939357a5 100644 --- a/packages/astro/src/content/loaders/types.ts +++ b/packages/astro/src/content/loaders/types.ts @@ -2,7 +2,11 @@ import type { FSWatcher } from 'vite'; import type { ZodSchema } from 'zod'; import type { AstroIntegrationLogger } from '../../core/logger/core.js'; import type { AstroConfig } from '../../types/public/config.js'; -import type { ContentEntryType } from '../../types/public/content.js'; +import type { + ContentEntryType, + LiveDataCollection, + LiveDataEntry, +} from '../../types/public/content.js'; import type { RenderedContent } from '../data-store.js'; import type { DataStore, MetaStore } from '../mutable-data-store.js'; @@ -53,3 +57,29 @@ export interface Loader { /** Optionally, define the schema of the data. Will be overridden by user-defined schema */ schema?: ZodSchema | Promise<ZodSchema> | (() => ZodSchema | Promise<ZodSchema>); } + +export interface LoadEntryContext<TEntryFilter = never> { + filter: TEntryFilter extends never ? { id: string } : TEntryFilter; +} + +export interface LoadCollectionContext<TCollectionFilter = unknown> { + filter?: TCollectionFilter; +} + +export interface LiveLoader< + TData extends Record<string, any> = Record<string, unknown>, + TEntryFilter extends Record<string, any> | never = never, + TCollectionFilter extends Record<string, any> | never = never, + TError extends Error = Error, +> { + /** Unique name of the loader, e.g. the npm package name */ + name: string; + /** Load a single entry */ + loadEntry: ( + context: LoadEntryContext<TEntryFilter>, + ) => Promise<LiveDataEntry<TData> | undefined | { error: TError }>; + /** Load a collection of entries */ + loadCollection: ( + context: LoadCollectionContext<TCollectionFilter>, + ) => Promise<LiveDataCollection<TData> | { error: TError }>; +} diff --git a/packages/astro/src/content/runtime.ts b/packages/astro/src/content/runtime.ts index 233c35c42..f2e02c6e1 100644 --- a/packages/astro/src/content/runtime.ts +++ b/packages/astro/src/content/runtime.ts @@ -5,7 +5,8 @@ import pLimit from 'p-limit'; import { ZodIssueCode, z } from 'zod'; import type { GetImageResult, ImageMetadata } from '../assets/types.js'; import { imageSrcToImportId } from '../assets/utils/resolveImports.js'; -import { AstroError, AstroErrorData, AstroUserError } from '../core/errors/index.js'; +import { defineCollection as defineCollectionOrig } from '../config/content.js'; +import { AstroError, AstroErrorData } from '../core/errors/index.js'; import { prependForwardSlash } from '../core/path.js'; import { @@ -19,38 +20,35 @@ import { render as serverRender, unescapeHTML, } from '../runtime/server/index.js'; -import { CONTENT_LAYER_TYPE, IMAGE_IMPORT_PREFIX } from './consts.js'; +import type { + LiveDataCollectionResult, + LiveDataEntry, + LiveDataEntryResult, +} from '../types/public/content.js'; +import { IMAGE_IMPORT_PREFIX, type LIVE_CONTENT_TYPE } from './consts.js'; import { type DataEntry, globalDataStore } from './data-store.js'; +import type { LiveLoader } from './loaders/types.js'; import type { ContentLookupMap } from './utils.js'; - +import { + LiveCollectionError, + LiveCollectionCacheHintError, + LiveEntryNotFoundError, + LiveCollectionValidationError, +} from './loaders/errors.js'; +export { + LiveCollectionError, + LiveCollectionCacheHintError, + LiveEntryNotFoundError, + LiveCollectionValidationError, +}; type LazyImport = () => Promise<any>; type GlobResult = Record<string, LazyImport>; type CollectionToEntryMap = Record<string, GlobResult>; type GetEntryImport = (collection: string, lookupId: string) => Promise<LazyImport>; - -export function getImporterFilename() { - // The 4th line in the stack trace should be the importer filename - const stackLine = new Error().stack?.split('\n')?.[3]; - if (!stackLine) { - return null; - } - // Extract the relative path from the stack line - const match = /\/(src\/.*?):\d+:\d+/.exec(stackLine); - return match?.[1] ?? null; -} - -export function defineCollection(config: any) { - if ('loader' in config) { - if (config.type && config.type !== CONTENT_LAYER_TYPE) { - throw new AstroUserError( - `Collections that use the Content Layer API must have a \`loader\` defined and no \`type\` set. Check your collection definitions in ${getImporterFilename() ?? 'your content config file'}.`, - ); - } - config.type = CONTENT_LAYER_TYPE; - } - if (!config.type) config.type = 'content'; - return config; -} +type LiveCollectionConfigMap = Record< + string, + { loader: LiveLoader; type: typeof LIVE_CONTENT_TYPE; schema?: z.ZodType } +>; export function createCollectionToGlobResultMap({ globResult, @@ -71,18 +69,74 @@ export function createCollectionToGlobResultMap({ return collectionToGlobResultMap; } +const cacheHintSchema = z.object({ + tags: z.array(z.string()).optional(), + maxAge: z.number().optional(), +}); + +async function parseLiveEntry( + entry: LiveDataEntry, + schema: z.ZodType, + collection: string, +): Promise<{ entry?: LiveDataEntry; error?: LiveCollectionError }> { + try { + const parsed = await schema.safeParseAsync(entry.data); + if (!parsed.success) { + return { + error: new LiveCollectionValidationError(collection, entry.id, parsed.error), + }; + } + if (entry.cacheHint) { + const cacheHint = cacheHintSchema.safeParse(entry.cacheHint); + + if (!cacheHint.success) { + return { + error: new LiveCollectionCacheHintError(collection, entry.id, cacheHint.error), + }; + } + entry.cacheHint = cacheHint.data; + } + return { + entry: { + ...entry, + data: parsed.data, + }, + }; + } catch (error) { + return { + error: new LiveCollectionError( + collection, + `Unexpected error parsing entry ${entry.id} in collection ${collection}`, + error as Error, + ), + }; + } +} + export function createGetCollection({ contentCollectionToEntryMap, dataCollectionToEntryMap, getRenderEntryImport, cacheEntriesByCollection, + liveCollections, }: { contentCollectionToEntryMap: CollectionToEntryMap; dataCollectionToEntryMap: CollectionToEntryMap; getRenderEntryImport: GetEntryImport; cacheEntriesByCollection: Map<string, any[]>; + liveCollections: LiveCollectionConfigMap; }) { - return async function getCollection(collection: string, filter?: (entry: any) => unknown) { + return async function getCollection( + collection: string, + filter?: ((entry: any) => unknown) | Record<string, unknown>, + ) { + if (collection in liveCollections) { + throw new AstroError({ + ...AstroErrorData.UnknownContentCollectionError, + message: `Collection "${collection}" is a live collection. Use getLiveCollection() instead of getCollection().`, + }); + } + const hasFilter = typeof filter === 'function'; const store = await globalDataStore.get(); let type: 'content' | 'data'; @@ -297,27 +351,29 @@ export function createGetEntry({ getEntryImport, getRenderEntryImport, collectionNames, + liveCollections, }: { getEntryImport: GetEntryImport; getRenderEntryImport: GetEntryImport; collectionNames: Set<string>; + liveCollections: LiveCollectionConfigMap; }) { return async function getEntry( // Can either pass collection and identifier as 2 positional args, // Or pass a single object with the collection and identifier as properties. // This means the first positional arg can have different shapes. collectionOrLookupObject: string | EntryLookupObject, - _lookupId?: string, + lookup?: string | Record<string, unknown>, ): Promise<ContentEntryResult | DataEntryResult | undefined> { - let collection: string, lookupId: string; + let collection: string, lookupId: string | Record<string, unknown>; if (typeof collectionOrLookupObject === 'string') { collection = collectionOrLookupObject; - if (!_lookupId) + if (!lookup) throw new AstroError({ ...AstroErrorData.UnknownContentCollectionError, message: '`getEntry()` requires an entry identifier as the second argument.', }); - lookupId = _lookupId; + lookupId = lookup; } else { collection = collectionOrLookupObject.collection; // Identifier could be `slug` for content entries, or `id` for data entries @@ -327,6 +383,18 @@ export function createGetEntry({ : collectionOrLookupObject.slug; } + if (collection in liveCollections) { + throw new AstroError({ + ...AstroErrorData.UnknownContentCollectionError, + message: `Collection "${collection}" is a live collection. Use getLiveEntry() instead of getEntry().`, + }); + } + if (typeof lookupId === 'object') { + throw new AstroError({ + ...AstroErrorData.UnknownContentCollectionError, + message: `The entry identifier must be a string. Received object.`, + }); + } const store = await globalDataStore.get(); if (store.hasCollection(collection)) { @@ -394,6 +462,186 @@ export function createGetEntries(getEntry: ReturnType<typeof createGetEntry>) { }; } +export function createGetLiveCollection({ + liveCollections, +}: { + liveCollections: LiveCollectionConfigMap; +}) { + return async function getLiveCollection( + collection: string, + filter?: Record<string, unknown>, + ): Promise<LiveDataCollectionResult> { + if (!(collection in liveCollections)) { + return { + error: new LiveCollectionError( + collection, + `Collection "${collection}" is not a live collection. Use getCollection() instead of getLiveCollection() to load regular content collections.`, + ), + }; + } + + try { + const context = { + filter, + }; + + const response = await ( + liveCollections[collection].loader as LiveLoader<any, any, Record<string, unknown>> + )?.loadCollection?.(context); + + // Check if loader returned an error + if (response && 'error' in response) { + return { error: response.error }; + } + + const { schema } = liveCollections[collection]; + + let processedEntries = response.entries; + if (schema) { + const entryResults = await Promise.all( + response.entries.map((entry) => parseLiveEntry(entry, schema, collection)), + ); + + // Check for parsing errors + for (const result of entryResults) { + if (result.error) { + // Return early on the first error + return { error: result.error }; + } + } + + processedEntries = entryResults.map((result) => result.entry!); + } + + let cacheHint = response.cacheHint; + if (cacheHint) { + const cacheHintResult = cacheHintSchema.safeParse(cacheHint); + + if (!cacheHintResult.success) { + return { + error: new LiveCollectionCacheHintError(collection, undefined, cacheHintResult.error), + }; + } + cacheHint = cacheHintResult.data; + } + + // Aggregate cache hints from individual entries if any + if (processedEntries.length > 0) { + const entryTags = new Set<string>(); + let minMaxAge: number | undefined; + + for (const entry of processedEntries) { + if (entry.cacheHint) { + if (entry.cacheHint.tags) { + entry.cacheHint.tags.forEach((tag) => entryTags.add(tag)); + } + if (typeof entry.cacheHint.maxAge === 'number') { + minMaxAge = + minMaxAge === undefined + ? entry.cacheHint.maxAge + : Math.min(minMaxAge, entry.cacheHint.maxAge); + } + } + } + + // Merge collection and entry cache hints + if (entryTags.size > 0 || minMaxAge !== undefined || cacheHint) { + const mergedCacheHint: any = {}; + if (cacheHint?.tags || entryTags.size > 0) { + mergedCacheHint.tags = [...(cacheHint?.tags || []), ...entryTags]; + } + if (cacheHint?.maxAge !== undefined || minMaxAge !== undefined) { + mergedCacheHint.maxAge = + cacheHint?.maxAge !== undefined && minMaxAge !== undefined + ? Math.min(cacheHint.maxAge, minMaxAge) + : (cacheHint?.maxAge ?? minMaxAge); + } + cacheHint = mergedCacheHint; + } + } + + return { + entries: processedEntries, + cacheHint, + }; + } catch (error) { + return { + error: new LiveCollectionError( + collection, + `Unexpected error loading collection ${collection}`, + error as Error, + ), + }; + } + }; +} + +export function createGetLiveEntry({ + liveCollections, +}: { + liveCollections: LiveCollectionConfigMap; +}) { + return async function getLiveEntry( + collection: string, + lookup: string | Record<string, unknown>, + ): Promise<LiveDataEntryResult> { + if (!(collection in liveCollections)) { + return { + error: new LiveCollectionError( + collection, + `Collection "${collection}" is not a live collection. Use getCollection() instead of getLiveEntry() to load regular content collections.`, + ), + }; + } + + try { + const lookupObject = { + filter: typeof lookup === 'string' ? { id: lookup } : lookup, + }; + + let entry = await ( + liveCollections[collection].loader as LiveLoader< + Record<string, unknown>, + Record<string, unknown> + > + )?.loadEntry?.(lookupObject); + + // Check if loader returned an error + if (entry && 'error' in entry) { + return { error: entry.error }; + } + + if (!entry) { + return { + error: new LiveEntryNotFoundError(collection, lookup), + }; + } + + const { schema } = liveCollections[collection]; + if (schema) { + const result = await parseLiveEntry(entry, schema, collection); + if (result.error) { + return { error: result.error }; + } + entry = result.entry!; + } + + return { + entry: entry, + cacheHint: entry.cacheHint, + }; + } catch (error) { + return { + error: new LiveCollectionError( + collection, + `Unexpected error loading entry ${collection} → ${typeof lookup === 'string' ? lookup : JSON.stringify(lookup)}`, + error as Error, + ), + }; + } + }; +} + type RenderResult = { Content: AstroComponentFactory; headings: MarkdownHeading[]; @@ -699,3 +947,15 @@ type PropagatedAssetsModule = { function isPropagatedAssetsModule(module: any): module is PropagatedAssetsModule { return typeof module === 'object' && module != null && '__astroPropagation' in module; } + +export function defineCollection(config: any) { + if (config.type === 'live') { + throw new AstroError({ + ...AstroErrorData.LiveContentConfigError, + message: AstroErrorData.LiveContentConfigError.message( + 'You must import defineCollection from "astro/config" to use live collections.', + ), + }); + } + return defineCollectionOrig(config); +} diff --git a/packages/astro/src/content/types-generator.ts b/packages/astro/src/content/types-generator.ts index c6a362a24..08250d9fe 100644 --- a/packages/astro/src/content/types-generator.ts +++ b/packages/astro/src/content/types-generator.ts @@ -7,7 +7,7 @@ import { type ViteDevServer, normalizePath } from 'vite'; import { type ZodSchema, z } from 'zod'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { AstroError } from '../core/errors/errors.js'; -import { AstroErrorData } from '../core/errors/index.js'; +import { AstroErrorData, AstroUserError } from '../core/errors/index.js'; import type { Logger } from '../core/logger/core.js'; import { isRelativePath } from '../core/path.js'; import type { AstroSettings } from '../types/astro.js'; @@ -16,6 +16,7 @@ import { COLLECTIONS_DIR, CONTENT_LAYER_TYPE, CONTENT_TYPES_FILE, + LIVE_CONTENT_TYPE, VIRTUAL_MODULE_ID, } from './consts.js'; import { @@ -53,6 +54,10 @@ type CollectionEntryMap = { | { type: 'data' | typeof CONTENT_LAYER_TYPE; entries: Record<string, DataEntryMetadata>; + } + | { + type: typeof LIVE_CONTENT_TYPE; + entries: Record<string, never>; }; }; @@ -493,6 +498,11 @@ async function writeContentFiles({ const collectionEntryKeys = Object.keys(collection.entries).sort(); const dataType = await typeForCollection(collectionConfig, collectionKey); switch (resolvedType) { + case LIVE_CONTENT_TYPE: + // This error should never be thrown, as it should have been caught earlier in the process + throw new AstroUserError( + `Invalid definition for collection ${collectionKey}: Live content collections must be defined in "src/live.config.ts"`, + ); case 'content': if (collectionEntryKeys.length === 0) { contentTypesStr += `${collectionKey}: Record<string, {\n id: string;\n slug: string;\n body: string;\n collection: ${collectionKey};\n data: ${dataType};\n render(): Render[".md"];\n}>;\n`; @@ -579,17 +589,28 @@ async function writeContentFiles({ contentPaths.config.url.pathname, ); + const liveConfigPathRelativeToCacheDir = contentPaths.liveConfig?.exists + ? normalizeConfigPath(settings.dotAstroDir.pathname, contentPaths.liveConfig.url.pathname) + : undefined; + for (const contentEntryType of contentEntryTypes) { if (contentEntryType.contentModuleTypes) { typeTemplateContent = contentEntryType.contentModuleTypes + '\n' + typeTemplateContent; } } - typeTemplateContent = typeTemplateContent.replace('// @@CONTENT_ENTRY_MAP@@', contentTypesStr); - typeTemplateContent = typeTemplateContent.replace('// @@DATA_ENTRY_MAP@@', dataTypesStr); - typeTemplateContent = typeTemplateContent.replace( - "'@@CONTENT_CONFIG_TYPE@@'", - contentConfig ? `typeof import(${configPathRelativeToCacheDir})` : 'never', - ); + typeTemplateContent = typeTemplateContent + .replace('// @@CONTENT_ENTRY_MAP@@', contentTypesStr) + .replace('// @@DATA_ENTRY_MAP@@', dataTypesStr) + .replace( + "'@@CONTENT_CONFIG_TYPE@@'", + contentConfig ? `typeof import(${configPathRelativeToCacheDir})` : 'never', + ) + .replace( + "'@@LIVE_CONTENT_CONFIG_TYPE@@'", + liveConfigPathRelativeToCacheDir + ? `typeof import(${liveConfigPathRelativeToCacheDir})` + : 'never', + ); // If it's the first time, we inject types the usual way. sync() will handle creating files and references. If it's not the first time, we just override the dts content if (settings.injectedTypes.some((t) => t.filename === CONTENT_TYPES_FILE)) { diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index 912ac58ed..e2400402f 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -22,6 +22,7 @@ import { CONTENT_MODULE_FLAG, DEFERRED_MODULE, IMAGE_IMPORT_PREFIX, + LIVE_CONTENT_TYPE, PROPAGATED_ASSET_FLAG, } from './consts.js'; import { glob } from './loaders/glob.js'; @@ -104,6 +105,11 @@ const collectionConfigParser = z.union([ /** deprecated */ _legacy: z.boolean().optional(), }), + z.object({ + type: z.literal(LIVE_CONTENT_TYPE), + schema: z.any().optional(), + loader: z.function(), + }), ]); const contentConfigParser = z.object({ @@ -557,7 +563,10 @@ async function autogenerateCollections({ const dataPattern = globWithUnderscoresIgnored('', dataExts); let usesContentLayer = false; for (const collectionName of Object.keys(collections)) { - if (collections[collectionName]?.type === 'content_layer') { + if ( + collections[collectionName]?.type === 'content_layer' || + collections[collectionName]?.type === 'live' + ) { usesContentLayer = true; // This is already a content layer, skip continue; @@ -705,13 +714,25 @@ export type ContentPaths = { exists: boolean; url: URL; }; + liveConfig: { + exists: boolean; + url: URL; + }; }; export function getContentPaths( - { srcDir, legacy, root }: Pick<AstroConfig, 'root' | 'srcDir' | 'legacy'>, + { + srcDir, + legacy, + root, + experimental, + }: Pick<AstroConfig, 'root' | 'srcDir' | 'legacy' | 'experimental'>, fs: typeof fsMod = fsMod, ): ContentPaths { - const configStats = search(fs, srcDir, legacy?.collections); + const configStats = searchConfig(fs, srcDir, legacy?.collections); + const liveConfigStats = experimental?.liveContentCollections + ? searchLiveConfig(fs, srcDir) + : { exists: false, url: new URL('./', srcDir) }; const pkgBase = new URL('../../', import.meta.url); return { root: new URL('./', root), @@ -720,9 +741,15 @@ export function getContentPaths( typesTemplate: new URL('templates/content/types.d.ts', pkgBase), virtualModTemplate: new URL('templates/content/module.mjs', pkgBase), config: configStats, + liveConfig: liveConfigStats, }; } -function search(fs: typeof fsMod, srcDir: URL, legacy?: boolean) { + +function searchConfig( + fs: typeof fsMod, + srcDir: URL, + legacy?: boolean, +): { exists: boolean; url: URL } { const paths = [ ...(legacy ? [] @@ -731,13 +758,23 @@ function search(fs: typeof fsMod, srcDir: URL, legacy?: boolean) { 'content/config.js', 'content/config.mts', 'content/config.ts', - ].map((p) => new URL(`./${p}`, srcDir)); - for (const file of paths) { + ]; + return search(fs, srcDir, paths); +} + +function searchLiveConfig(fs: typeof fsMod, srcDir: URL): { exists: boolean; url: URL } { + const paths = ['live.config.mjs', 'live.config.js', 'live.config.mts', 'live.config.ts']; + return search(fs, srcDir, paths); +} + +function search(fs: typeof fsMod, srcDir: URL, paths: string[]): { exists: boolean; url: URL } { + const urls = paths.map((p) => new URL(`./${p}`, srcDir)); + for (const file of urls) { if (fs.existsSync(file)) { return { exists: true, url: file }; } } - return { exists: false, url: paths[0] }; + return { exists: false, url: urls[0] }; } /** diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index ff0435844..d52241aa0 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -247,7 +247,14 @@ async function generateContentEntryFile({ .replace("'@@CONTENT_ENTRY_GLOB_PATH@@'", contentEntryGlobResult) .replace("'@@DATA_ENTRY_GLOB_PATH@@'", dataEntryGlobResult) .replace("'@@RENDER_ENTRY_GLOB_PATH@@'", renderEntryGlobResult) - .replace('/* @@LOOKUP_MAP_ASSIGNMENT@@ */', `lookupMap = ${JSON.stringify(lookupMap)};`); + .replace('/* @@LOOKUP_MAP_ASSIGNMENT@@ */', `lookupMap = ${JSON.stringify(lookupMap)};`) + .replace( + '/* @@LIVE_CONTENT_CONFIG@@ */', + contentPaths.liveConfig.exists + ? // Dynamic import so it extracts the chunk and avoids a circular import + `const liveCollections = (await import(${JSON.stringify(fileURLToPath(contentPaths.liveConfig.url))})).collections;` + : 'const liveCollections = {};', + ); } return virtualModContents; @@ -258,13 +265,7 @@ async function generateContentEntryFile({ * This is used internally to resolve entry imports when using `getEntry()`. * @see `templates/content/module.mjs` */ -async function generateLookupMap({ - settings, - fs, -}: { - settings: AstroSettings; - fs: typeof nodeFs; -}) { +async function generateLookupMap({ settings, fs }: { settings: AstroSettings; fs: typeof nodeFs }) { const { root } = settings.config; const contentPaths = getContentPaths(settings.config); const relContentDir = rootRelativePath(root, contentPaths.contentDir, false); diff --git a/packages/astro/src/core/config/schemas/base.ts b/packages/astro/src/core/config/schemas/base.ts index 9f6d36653..3eb0d6235 100644 --- a/packages/astro/src/core/config/schemas/base.ts +++ b/packages/astro/src/core/config/schemas/base.ts @@ -101,6 +101,7 @@ export const ASTRO_CONFIG_DEFAULTS = { responsiveImages: false, headingIdCompat: false, preserveScriptOrder: false, + liveContentCollections: false, csp: false, }, } satisfies AstroUserConfig & { server: { open: boolean } }; @@ -479,6 +480,10 @@ export const AstroConfigSchema = z.object({ .optional() .default(ASTRO_CONFIG_DEFAULTS.experimental.preserveScriptOrder), fonts: z.array(z.union([localFontFamilySchema, remoteFontFamilySchema])).optional(), + liveContentCollections: z + .boolean() + .optional() + .default(ASTRO_CONFIG_DEFAULTS.experimental.liveContentCollections), csp: z .union([ z.boolean().optional().default(ASTRO_CONFIG_DEFAULTS.experimental.csp), diff --git a/packages/astro/src/core/errors/errors-data.ts b/packages/astro/src/core/errors/errors-data.ts index 28513963a..2190453bb 100644 --- a/packages/astro/src/core/errors/errors-data.ts +++ b/packages/astro/src/core/errors/errors-data.ts @@ -1643,8 +1643,9 @@ export const InvalidContentEntryDataError = { title: 'Content entry data does not match schema.', message(collection: string, entryId: string, error: ZodError) { return [ - `**${String(collection)} → ${String(entryId)}** data does not match collection schema.`, - ...error.errors.map((zodError) => zodError.message), + `**${String(collection)} → ${String(entryId)}** data does not match collection schema.\n`, + ...error.errors.map((zodError) => ` **${zodError.path.join('.')}**: ${zodError.message}`), + '', ].join('\n'); }, hint: 'See https://docs.astro.build/en/guides/content-collections/ for more information on content schemas.', @@ -1694,8 +1695,9 @@ export const ContentEntryDataError = { title: 'Content entry data does not match schema.', message(collection: string, entryId: string, error: ZodError) { return [ - `**${String(collection)} → ${String(entryId)}** data does not match collection schema.`, - ...error.errors.map((zodError) => zodError.message), + `**${String(collection)} → ${String(entryId)}** data does not match collection schema.\n`, + ...error.errors.map((zodError) => ` **${zodError.path.join('.')}**: ${zodError.message}`), + '', ].join('\n'); }, hint: 'See https://docs.astro.build/en/guides/content-collections/ for more information on content schemas.', @@ -1705,6 +1707,25 @@ export const ContentEntryDataError = { * @docs * @message * **Example error message:**<br/> + * The schema cannot be a function for live collections. Please use a schema object instead. Check your collection definitions in your live content config file. + * @description + * Error in live content config. + * @see + * - [Experimental live content](https://astro.build/en/reference/experimental-flags/live-content-collections/) + */ + +export const LiveContentConfigError = { + name: 'LiveContentConfigError', + title: 'Error in live content config.', + message: (error: string, filename?: string) => + `${error} Check your collection definitions in ${filename ?? 'your live content config file'}.`, + hint: 'See https://docs.astro.build/en/reference/experimental-flags/live-content-collections/ for more information on live content collections.', +} satisfies ErrorData; + +/** + * @docs + * @message + * **Example error message:**<br/> * The loader for **blog** returned invalid data.<br/> * Object is missing required property "id". * @description diff --git a/packages/astro/src/core/errors/errors.ts b/packages/astro/src/core/errors/errors.ts index 88ff70de3..71c4e0cf6 100644 --- a/packages/astro/src/core/errors/errors.ts +++ b/packages/astro/src/core/errors/errors.ts @@ -27,7 +27,7 @@ type ErrorTypes = | 'AggregateError'; export function isAstroError(e: unknown): e is AstroError { - return e instanceof AstroError; + return e instanceof AstroError || AstroError.is(e); } export class AstroError extends Error { diff --git a/packages/astro/src/types/public/config.ts b/packages/astro/src/types/public/config.ts index b1bfbbe22..284ad9188 100644 --- a/packages/astro/src/types/public/config.ts +++ b/packages/astro/src/types/public/config.ts @@ -2524,6 +2524,17 @@ export interface ViteUserConfig extends OriginalViteUserConfig { * */ preserveScriptOrder?: boolean; + + /** + * @name experimental.liveContentCollections + * @type {boolean} + * @default `false` + * @version 5.x + * @description + * Enables the use of live content collections. + * + */ + liveContentCollections?: boolean; }; } diff --git a/packages/astro/src/types/public/content.ts b/packages/astro/src/types/public/content.ts index 171c8d550..f81e6b377 100644 --- a/packages/astro/src/types/public/content.ts +++ b/packages/astro/src/types/public/content.ts @@ -3,6 +3,7 @@ import type * as rollup from 'rollup'; import type { DataEntry, RenderedContent } from '../../content/data-store.js'; import type { AstroComponentFactory } from '../../runtime/server/index.js'; import type { AstroConfig } from './config.js'; +import type { LiveCollectionError } from '../../content/loaders/errors.js'; export interface AstroInstance { file: string; @@ -124,3 +125,45 @@ export interface DataEntryType { } export type GetDataEntryInfoReturnType = { data: Record<string, unknown>; rawData?: string }; + +export interface CacheHint { + /** Cache tags */ + tags?: Array<string>; + /** Maximum age of the response in seconds */ + maxAge?: number; +} + +export interface LiveDataEntry<TData extends Record<string, unknown> = Record<string, unknown>> { + /** The ID of the entry. Unique per collection. */ + id: string; + /** The parsed entry data */ + data: TData; + /** A hint for how to cache this entry */ + cacheHint?: CacheHint; +} + +export interface LiveDataCollection< + TData extends Record<string, unknown> = Record<string, unknown>, +> { + entries: Array<LiveDataEntry<TData>>; + /** A hint for how to cache this collection. Individual entries can also have cache hints */ + cacheHint?: CacheHint; +} + +export interface LiveDataCollectionResult< + TData extends Record<string, unknown> = Record<string, unknown>, + TError extends Error = Error, +> { + entries?: Array<LiveDataEntry<TData>>; + error?: TError | LiveCollectionError + cacheHint?: CacheHint; +} + +export interface LiveDataEntryResult< + TData extends Record<string, unknown> = Record<string, unknown>, + TError extends Error = Error, +> { + entry?: LiveDataEntry<TData>; + error?: TError | LiveCollectionError; + cacheHint?: CacheHint; +} diff --git a/packages/astro/templates/content/module.mjs b/packages/astro/templates/content/module.mjs index 7947574c3..1baac47c7 100644 --- a/packages/astro/templates/content/module.mjs +++ b/packages/astro/templates/content/module.mjs @@ -6,12 +6,16 @@ import { createGetEntries, createGetEntry, createGetEntryBySlug, + createGetLiveCollection, + createGetLiveEntry, createReference, } from 'astro/content/runtime'; export { defineCollection, renderEntry as render } from 'astro/content/runtime'; export { z } from 'astro/zod'; +/* @@LIVE_CONTENT_CONFIG@@ */ + const contentDir = '@@CONTENT_DIR@@'; const contentEntryGlob = '@@CONTENT_ENTRY_GLOB_PATH@@'; @@ -56,12 +60,14 @@ export const getCollection = createGetCollection({ dataCollectionToEntryMap, getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap), cacheEntriesByCollection, + liveCollections, }); export const getEntry = createGetEntry({ getEntryImport: createGlobLookup(collectionToEntryMap), getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap), collectionNames, + liveCollections, }); export const getEntryBySlug = createGetEntryBySlug({ @@ -80,3 +86,11 @@ export const getDataEntryById = createGetDataEntryById({ export const getEntries = createGetEntries(getEntry); export const reference = createReference({ lookupMap }); + +export const getLiveCollection = createGetLiveCollection({ + liveCollections, +}); + +export const getLiveEntry = createGetLiveEntry({ + liveCollections, +}); diff --git a/packages/astro/templates/content/types.d.ts b/packages/astro/templates/content/types.d.ts index 2d1058d27..d441097d8 100644 --- a/packages/astro/templates/content/types.d.ts +++ b/packages/astro/templates/content/types.d.ts @@ -45,6 +45,10 @@ declare module 'astro:content' { collection: C; slug: E; }; + export type ReferenceLiveEntry<C extends keyof LiveContentConfig['collections']> = { + collection: C; + id: string; + }; /** @deprecated Use `getEntry` instead. */ export function getEntryBySlug< @@ -73,6 +77,11 @@ declare module 'astro:content' { filter?: (entry: CollectionEntry<C>) => unknown, ): Promise<CollectionEntry<C>[]>; + export function getLiveCollection<C extends keyof LiveContentConfig['collections']>( + collection: C, + filter?: LiveLoaderCollectionFilterType<C>, + ): Promise<import('astro').LiveDataCollectionResult<LiveLoaderDataType<C>, LiveLoaderErrorType<C>>>; + export function getEntry< C extends keyof ContentEntryMap, E extends ValidContentEntrySlug<C> | (string & {}), @@ -109,6 +118,10 @@ declare module 'astro:content' { ? Promise<DataEntryMap[C][E]> | undefined : Promise<DataEntryMap[C][E]> : Promise<CollectionEntry<C> | undefined>; + export function getLiveEntry<C extends keyof LiveContentConfig['collections']>( + collection: C, + filter: string | LiveLoaderEntryFilterType<C>, + ): Promise<import('astro').LiveDataEntryResult<LiveLoaderDataType<C>, LiveLoaderErrorType<C>>>; /** Resolve an array of entry references from the same collection */ export function getEntries<C extends keyof ContentEntryMap>( @@ -152,5 +165,32 @@ declare module 'astro:content' { type AnyEntryMap = ContentEntryMap & DataEntryMap; + type ExtractLoaderTypes<T> = T extends import('astro/loaders').LiveLoader< + infer TData, + infer TEntryFilter, + infer TCollectionFilter, + infer TError + > + ? { data: TData; entryFilter: TEntryFilter; collectionFilter: TCollectionFilter; error: TError } + : { data: never; entryFilter: never; collectionFilter: never; error: never }; + type ExtractDataType<T> = ExtractLoaderTypes<T>['data']; + type ExtractEntryFilterType<T> = ExtractLoaderTypes<T>['entryFilter']; + type ExtractCollectionFilterType<T> = ExtractLoaderTypes<T>['collectionFilter']; + type ExtractErrorType<T> = ExtractLoaderTypes<T>['error']; + + type LiveLoaderDataType<C extends keyof LiveContentConfig['collections']> = + LiveContentConfig['collections'][C]['schema'] extends undefined + ? ExtractDataType<LiveContentConfig['collections'][C]['loader']> + : import('astro/zod').infer< + Exclude<LiveContentConfig['collections'][C]['schema'], undefined> + >; + type LiveLoaderEntryFilterType<C extends keyof LiveContentConfig['collections']> = + ExtractEntryFilterType<LiveContentConfig['collections'][C]['loader']>; + type LiveLoaderCollectionFilterType<C extends keyof LiveContentConfig['collections']> = + ExtractCollectionFilterType<LiveContentConfig['collections'][C]['loader']>; + type LiveLoaderErrorType<C extends keyof LiveContentConfig['collections']> = + ExtractErrorType<LiveContentConfig['collections'][C]['loader']>; + export type ContentConfig = '@@CONTENT_CONFIG_TYPE@@'; + export type LiveContentConfig = '@@LIVE_CONTENT_CONFIG_TYPE@@'; } diff --git a/packages/astro/test/fixtures/live-loaders/astro.config.mjs b/packages/astro/test/fixtures/live-loaders/astro.config.mjs new file mode 100644 index 000000000..44512231b --- /dev/null +++ b/packages/astro/test/fixtures/live-loaders/astro.config.mjs @@ -0,0 +1,14 @@ +// @ts-check +import { defineConfig } from 'astro/config'; + +import node from '@astrojs/node'; + +// https://astro.build/config +export default defineConfig({ + adapter: node({ + mode: 'standalone' + }), + experimental: { + liveContentCollections: true + } +}); diff --git a/packages/astro/test/fixtures/live-loaders/package.json b/packages/astro/test/fixtures/live-loaders/package.json new file mode 100644 index 000000000..6be0ee921 --- /dev/null +++ b/packages/astro/test/fixtures/live-loaders/package.json @@ -0,0 +1,15 @@ +{ + "name": "@test/live-loaders", + "type": "module", + "version": "0.0.1", + "scripts": { + "dev": "astro dev", + "build": "astro build", + "preview": "astro preview", + "astro": "astro" + }, + "dependencies": { + "@astrojs/node": "workspace:*", + "astro": "workspace:*" + } +} diff --git a/packages/astro/test/fixtures/live-loaders/src/content.config.ts b/packages/astro/test/fixtures/live-loaders/src/content.config.ts new file mode 100644 index 000000000..f3a629cd1 --- /dev/null +++ b/packages/astro/test/fixtures/live-loaders/src/content.config.ts @@ -0,0 +1,5 @@ +import { defineCollection } from "astro:content"; +const something = defineCollection({ + loader: () => ([]) +}) +export const collections = { something }; diff --git a/packages/astro/test/fixtures/live-loaders/src/live.config.ts b/packages/astro/test/fixtures/live-loaders/src/live.config.ts new file mode 100644 index 000000000..9fde1c154 --- /dev/null +++ b/packages/astro/test/fixtures/live-loaders/src/live.config.ts @@ -0,0 +1,86 @@ +import { defineCollection } from 'astro/config'; +import { z } from 'astro/zod'; +import type { LiveLoader } from 'astro/loaders'; + +type Entry = { + title: string; + age?: number; +}; + +interface CollectionFilter { + addToAge?: number; + returnInvalid?: boolean; +} + +type EntryFilter = { + id: keyof typeof entries; + addToAge?: number; +}; + +const entries = { + '123': { id: '123', data: { title: 'Page 123', age: 10 } }, + '456': { id: '456', data: { title: 'Page 456', age: 20 } }, + '789': { id: '789', data: { title: 'Page 789', age: 30 } }, +}; + +class CustomError extends Error { + constructor(message: string) { + super(message); + this.name = 'CustomError'; + } +} + +const loader: LiveLoader<Entry, EntryFilter, CollectionFilter, CustomError> = { + name: 'test-loader', + loadEntry: async ({ filter }) => { + const entry = entries[filter.id]; + if (!entry) { + return { + error: new CustomError(`Entry ${filter.id} not found`), + }; + } + return { + ...entry, + data: { + title: entry.data.title, + age: filter?.addToAge + ? entry.data.age + ? entry.data.age + filter.addToAge + : filter.addToAge + : entry.data.age, + }, + cacheHint: { + tags: [`page:${filter.id}`], + maxAge: 60, + }, + }; + }, + loadCollection: async ({filter}) => { + return { + entries: filter?.addToAge + ? Object.values(entries).map((entry) => ({ + ...entry, + data: { + title: filter.returnInvalid ? 99 as any : entry.data.title, + age: entry.data.age ? entry.data.age + filter!.addToAge! : undefined, + }, + })) + : Object.values(entries), + cacheHint: { + tags: ['page'], + maxAge: 60, + }, + }; + }, +}; + +const liveStuff = defineCollection({ + type: 'live', + loader, + schema: z.object({ + title: z.string(), + age: z.number().optional(), + }), +}); + +export const collections = { liveStuff }; diff --git a/packages/astro/test/fixtures/live-loaders/src/pages/api.ts b/packages/astro/test/fixtures/live-loaders/src/pages/api.ts new file mode 100644 index 000000000..86690ed8b --- /dev/null +++ b/packages/astro/test/fixtures/live-loaders/src/pages/api.ts @@ -0,0 +1,23 @@ +import type { APIRoute } from 'astro'; +import { getLiveCollection, getLiveEntry } from 'astro:content'; + +export const prerender = false; + +export const GET: APIRoute = async ({ url }) => { + const addToAge = url.searchParams.get('addToAge'); + const returnInvalid = url.searchParams.has('returnInvalid'); + const filter = addToAge ? { addToAge: parseInt(addToAge), returnInvalid } : undefined; + const { error, entries, cacheHint } = await getLiveCollection('liveStuff', filter); + const entryByString = await getLiveEntry('liveStuff', '123'); + const entryByObject = await getLiveEntry('liveStuff', { id: '456', ...filter }); + + return Response.json({ + collection: { + cacheHint, + entries, + error: error ? { ...error, message: error.message } : undefined, + }, + entryByObject, + entryByString, + }); +}; diff --git a/packages/astro/test/fixtures/live-loaders/src/pages/index.astro b/packages/astro/test/fixtures/live-loaders/src/pages/index.astro new file mode 100644 index 000000000..0347a3334 --- /dev/null +++ b/packages/astro/test/fixtures/live-loaders/src/pages/index.astro @@ -0,0 +1,32 @@ +--- +import { getLiveCollection } from "astro:content"; + +const collection = await getLiveCollection("liveStuff") + +if(collection.error) { + throw collection.error; +} + +export const prerender = false; + +--- + +<html lang="en"> + <head> + <meta charset="utf-8" /> + <link rel="icon" type="image/svg+xml" href="/favicon.svg" /> + <meta name="viewport" content="width=device-width" /> + <meta name="generator" content={Astro.generator} /> + <title>Astro</title> + </head> + <body> + <h1>Astro</h1> + <ul> + {collection.entries?.map((item) => ( + <li> + {item.data.title} + </li> + ))} + </ul> + </body> +</html> diff --git a/packages/astro/test/fixtures/live-loaders/src/pages/more.astro b/packages/astro/test/fixtures/live-loaders/src/pages/more.astro new file mode 100644 index 000000000..52088b5e1 --- /dev/null +++ b/packages/astro/test/fixtures/live-loaders/src/pages/more.astro @@ -0,0 +1,20 @@ +--- +import { getLiveEntry } from "astro:content"; + +const { entry } = await getLiveEntry("liveStuff", "123") +export const prerender = false; + +--- + +<html lang="en"> + <head> + <meta charset="utf-8" /> + <link rel="icon" type="image/svg+xml" href="/favicon.svg" /> + <meta name="viewport" content="width=device-width" /> + <meta name="generator" content={Astro.generator} /> + <title>Astro</title> + </head> + <body> + <h1>{entry?.data.title}</h1> + </body> +</html> diff --git a/packages/astro/test/fixtures/live-loaders/src/pages/test-old-api.ts b/packages/astro/test/fixtures/live-loaders/src/pages/test-old-api.ts new file mode 100644 index 000000000..419f53f86 --- /dev/null +++ b/packages/astro/test/fixtures/live-loaders/src/pages/test-old-api.ts @@ -0,0 +1,21 @@ +import type { APIRoute } from 'astro'; +import { getCollection } from 'astro:content'; + +export const prerender = false; + +export const GET: APIRoute = async () => { + try { + // @ts-ignore This should throw an error because liveStuff is a live collection + const collection = await getCollection('liveStuff'); + return Response.json({ collection }); + } catch (error: any) { + return Response.json( + { + error: error.message + }, + { + status: 500, + }, + ); + } +}; diff --git a/packages/astro/test/fixtures/live-loaders/tsconfig.json b/packages/astro/test/fixtures/live-loaders/tsconfig.json new file mode 100644 index 000000000..8bf91d3bb --- /dev/null +++ b/packages/astro/test/fixtures/live-loaders/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "astro/tsconfigs/strict", + "include": [".astro/types.d.ts", "**/*"], + "exclude": ["dist"] +} diff --git a/packages/astro/test/live-loaders.test.js b/packages/astro/test/live-loaders.test.js new file mode 100644 index 000000000..5e8480459 --- /dev/null +++ b/packages/astro/test/live-loaders.test.js @@ -0,0 +1,212 @@ +// @ts-check +import assert from 'node:assert/strict'; +import { Writable } from 'node:stream'; +import { after, before, describe, it } from 'node:test'; +import { Logger } from '../dist/core/logger/core.js'; + +import testAdapter from './test-adapter.js'; +import { loadFixture } from './test-utils.js'; +describe('Live content collections', () => { + let fixture; + before(async () => { + fixture = await loadFixture({ + root: './fixtures/live-loaders/', + adapter: testAdapter(), + }); + }); + describe('Dev', () => { + let devServer; + const logs = []; + before(async () => { + devServer = await fixture.startDevServer({ + logger: new Logger({ + level: 'info', + dest: new Writable({ + objectMode: true, + write(event, _, callback) { + logs.push(event); + callback(); + }, + }), + }), + }); + }); + + after(async () => { + devServer?.stop(); + }); + + it('can load live data', async () => { + const res = await fixture.fetch('/api/'); + assert.equal(res.status, 200); + const data = await res.json(); + assert.deepEqual(data.entryByString, { + entry: { + id: '123', + data: { title: 'Page 123', age: 10 }, + cacheHint: { + tags: [`page:123`], + maxAge: 60, + }, + }, + cacheHint: { + tags: [`page:123`], + maxAge: 60, + }, + }); + assert.deepEqual(data.entryByObject, { + entry: { + id: '456', + data: { title: 'Page 456', age: 20 }, + cacheHint: { + tags: [`page:456`], + maxAge: 60, + }, + }, + cacheHint: { + tags: [`page:456`], + maxAge: 60, + }, + }); + assert.deepEqual(data.collection, { + entries: [ + { + id: '123', + data: { title: 'Page 123', age: 10 }, + }, + { + id: '456', + data: { title: 'Page 456', age: 20 }, + }, + { + id: '789', + data: { title: 'Page 789', age: 30 }, + }, + ], + cacheHint: { + tags: ['page'], + maxAge: 60, + }, + }); + }); + + it('can load live data with dynamic filtering', async () => { + const res = await fixture.fetch('/api/?addToAge=5'); + assert.equal(res.status, 200); + const data = await res.json(); + assert.deepEqual( + data.entryByObject, + { + entry: { + id: '456', + data: { title: 'Page 456', age: 25 }, + cacheHint: { + tags: [`page:456`], + maxAge: 60, + }, + }, + cacheHint: { + tags: [`page:456`], + maxAge: 60, + }, + }, + 'passes dynamic filter to getEntry', + ); + assert.deepEqual( + data.collection.entries, + [ + { + id: '123', + data: { title: 'Page 123', age: 15 }, + }, + { + id: '456', + data: { title: 'Page 456', age: 25 }, + }, + { + id: '789', + data: { title: 'Page 789', age: 35 }, + }, + ], + 'passes dynamic filter to getCollection', + ); + }); + + it('returns an error for invalid data', async () => { + const res = await fixture.fetch('/api/?returnInvalid=true&addToAge=1'); + const data = await res.json(); + assert.ok(data.collection.error.message.includes('data does not match the collection schema')); + assert.equal(data.collection.error.name, 'LiveCollectionValidationError'); + }); + + it('old API throws helpful errors for live collections', async () => { + const response = await fixture.fetch('/test-old-api'); + const data = await response.json(); + assert.ok(data.error.includes('Use getLiveCollection() instead of getCollection()')); + }); + }); + + describe('SSR', () => { + let app; + + before(async () => { + await fixture.build(); + app = await fixture.loadTestAdapterApp(); + }); + + it('loads live data', async () => { + const req = new Request('http://example.com/api/'); + const response = await app.render(req); + assert.ok(response.ok); + assert.equal(response.status, 200); + const data = await response.json(); + assert.deepEqual(data.entryByString, { + entry: { + id: '123', + data: { title: 'Page 123', age: 10 }, + cacheHint: { + tags: [`page:123`], + maxAge: 60, + }, + }, + cacheHint: { + tags: [`page:123`], + maxAge: 60, + }, + }); + }); + it('loads live data with dynamic filtering', async () => { + const request = new Request('http://example.com/api/?addToAge=5'); + const response = await app.render(request); + assert.ok(response.ok); + assert.equal(response.status, 200); + const data = await response.json(); + assert.deepEqual( + data.entryByObject, + { + entry: { + id: '456', + data: { title: 'Page 456', age: 25 }, + cacheHint: { + tags: [`page:456`], + maxAge: 60, + }, + }, + cacheHint: { + tags: [`page:456`], + maxAge: 60, + }, + }, + 'passes dynamic filter to getEntry', + ); + }); + + it('old API throws helpful errors for live collections', async () => { + const request = new Request('http://example.com/test-old-api'); + const response = await app.render(request); + assert.equal(response.status, 500); + const data = await response.json(); + assert.ok(data.error.includes('Use getLiveCollection() instead of getCollection()')); + }); + }); +}); diff --git a/packages/astro/types/content.d.ts b/packages/astro/types/content.d.ts index 19200d475..78b3ca616 100644 --- a/packages/astro/types/content.d.ts +++ b/packages/astro/types/content.d.ts @@ -1,85 +1,17 @@ declare module 'astro:content' { export { z } from 'astro/zod'; + export type { + ImageFunction, + DataEntry, + DataStore, + MetaStore, + BaseSchema, + SchemaContext, + } from 'astro/config'; - // This needs to be in sync with ImageMetadata - export type ImageFunction = () => import('astro/zod').ZodObject<{ - src: import('astro/zod').ZodString; - width: import('astro/zod').ZodNumber; - height: import('astro/zod').ZodNumber; - format: import('astro/zod').ZodUnion< - [ - import('astro/zod').ZodLiteral<'png'>, - import('astro/zod').ZodLiteral<'jpg'>, - import('astro/zod').ZodLiteral<'jpeg'>, - import('astro/zod').ZodLiteral<'tiff'>, - import('astro/zod').ZodLiteral<'webp'>, - import('astro/zod').ZodLiteral<'gif'>, - import('astro/zod').ZodLiteral<'svg'>, - import('astro/zod').ZodLiteral<'avif'>, - ] - >; - }>; - - export interface DataEntry { - id: string; - data: Record<string, unknown>; - filePath?: string; - body?: string; - } - - export interface DataStore { - get: (key: string) => DataEntry; - entries: () => Array<[id: string, DataEntry]>; - set: (key: string, data: Record<string, unknown>, body?: string, filePath?: string) => void; - values: () => Array<DataEntry>; - keys: () => Array<string>; - delete: (key: string) => void; - clear: () => void; - has: (key: string) => boolean; - } - - export interface MetaStore { - get: (key: string) => string | undefined; - set: (key: string, value: string) => void; - delete: (key: string) => void; - has: (key: string) => boolean; - } - - export type BaseSchema = import('astro/zod').ZodType; - - export type SchemaContext = { image: ImageFunction }; - - type ContentLayerConfig<S extends BaseSchema, TData extends { id: string } = { id: string }> = { - type?: 'content_layer'; - schema?: S | ((context: SchemaContext) => S); - loader: - | import('astro/loaders').Loader - | (() => - | Array<TData> - | Promise<Array<TData>> - | Record<string, Omit<TData, 'id'> & { id?: string }> - | Promise<Record<string, Omit<TData, 'id'> & { id?: string }>>); - }; - - type DataCollectionConfig<S extends BaseSchema> = { - type: 'data'; - schema?: S | ((context: SchemaContext) => S); - }; - - type ContentCollectionConfig<S extends BaseSchema> = { - type?: 'content'; - schema?: S | ((context: SchemaContext) => S); - loader?: never; - }; - - export type CollectionConfig<S extends BaseSchema> = - | ContentCollectionConfig<S> - | DataCollectionConfig<S> - | ContentLayerConfig<S>; - - export function defineCollection<S extends BaseSchema>( - input: CollectionConfig<S>, - ): CollectionConfig<S>; + export function defineCollection<S extends import('astro/config').BaseSchema>( + input: import('astro/config').BaseCollectionConfig<S>, + ): import('astro/config').BaseCollectionConfig<S>; /** Run `astro dev` or `astro sync` to generate high fidelity types */ export const getEntryBySlug: (...args: any[]) => any; @@ -106,4 +38,8 @@ declare module 'astro:content' { export type ContentConfig = any; /** Run `astro dev` or `astro sync` to generate high fidelity types */ export const render: (entry: any) => any; + /** Run `astro dev` or `astro sync` to generate high fidelity types */ + export const getLiveCollection: (...args: any[]) => any; + /** Run `astro dev` or `astro sync` to generate high fidelity types */ + export const getLiveEntry: (...args: any[]) => any; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a0fabafe..1f614ef1d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3426,6 +3426,15 @@ importers: specifier: workspace:* version: link:../../.. + packages/astro/test/fixtures/live-loaders: + dependencies: + '@astrojs/node': + specifier: workspace:* + version: link:../../../../integrations/node + astro: + specifier: workspace:* + version: link:../../.. + packages/astro/test/fixtures/markdown: dependencies: '@astrojs/preact': @@ -5591,7 +5600,7 @@ importers: dependencies: '@vitejs/plugin-react': specifier: ^4.5.0 - version: 4.5.1(vite@6.3.5(@types/node@22.13.1)(jiti@2.4.2)(lightningcss@1.29.2)(sass@1.86.3)(yaml@2.5.1)) + version: 4.5.0(vite@6.3.5(@types/node@22.13.1)(jiti@2.4.2)(lightningcss@1.29.2)(sass@1.86.3)(yaml@2.5.1)) ultrahtml: specifier: ^1.6.0 version: 1.6.0 @@ -8630,8 +8639,8 @@ packages: '@vercel/routing-utils@5.0.4': resolution: {integrity: sha512-4ke67zkXVi2fRZdoYckABcsSkRC9CnrdadOGxoS/Bk22+ObHjGQWvUHExRSXh339anwu9YY7ZacNSGH4gUnTQA==} - '@vitejs/plugin-react@4.5.1': - resolution: {integrity: sha512-uPZBqSI0YD4lpkIru6M35sIfylLGTyhGHvDZbNLuMA73lMlwJKz5xweH7FajfcCAc2HnINciejA9qTz0dr0M7A==} + '@vitejs/plugin-react@4.5.0': + resolution: {integrity: sha512-JuLWaEqypaJmOJPLWwO335Ig6jSgC1FTONCWAxnqcQthLTK/Yc9aH6hr9z/87xciejbQcnP3GnA1FWUSWeXaeg==} engines: {node: ^14.18.0 || >=16.0.0} peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 @@ -10694,6 +10703,7 @@ packages: libsql@0.5.4: resolution: {integrity: sha512-GEFeWca4SDAQFxjHWJBE6GK52LEtSskiujbG3rqmmeTO9t4sfSBKIURNLLpKDDF7fb7jmTuuRkDAn9BZGITQNw==} + cpu: [x64, arm64, wasm32] os: [darwin, linux, win32] lightningcss-darwin-arm64@1.29.2: @@ -15749,7 +15759,7 @@ snapshots: optionalDependencies: ajv: 6.12.6 - '@vitejs/plugin-react@4.5.1(vite@6.3.5(@types/node@22.13.1)(jiti@2.4.2)(lightningcss@1.29.2)(sass@1.86.3)(yaml@2.5.1))': + '@vitejs/plugin-react@4.5.0(vite@6.3.5(@types/node@22.13.1)(jiti@2.4.2)(lightningcss@1.29.2)(sass@1.86.3)(yaml@2.5.1))': dependencies: '@babel/core': 7.27.1 '@babel/plugin-transform-react-jsx-self': 7.25.9(@babel/core@7.27.1) diff --git a/tsconfig.base.json b/tsconfig.base.json index 432d3c353..2141875ed 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -14,4 +14,4 @@ "noUnusedLocals": true, "noUnusedParameters": true } -} +}
\ No newline at end of file |