diff options
| -rw-r--r-- | .changeset/sixty-masks-lie.md | 5 | ||||
| -rw-r--r-- | packages/astro/package.json | 1 | ||||
| -rw-r--r-- | packages/astro/src/content/content-layer.ts | 66 | ||||
| -rw-r--r-- | packages/astro/src/content/loaders/types.ts | 1 | ||||
| -rw-r--r-- | packages/astro/src/core/dev/restart.ts | 4 | ||||
| -rw-r--r-- | packages/astro/src/types/public/content.ts | 5 |
6 files changed, 46 insertions, 36 deletions
diff --git a/.changeset/sixty-masks-lie.md b/.changeset/sixty-masks-lie.md new file mode 100644 index 000000000..d9b8d7a52 --- /dev/null +++ b/.changeset/sixty-masks-lie.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Refactors content layer sync to use a queue diff --git a/packages/astro/package.json b/packages/astro/package.json index a448146ee..897917eba 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -150,6 +150,7 @@ "esbuild": "^0.21.5", "estree-walker": "^3.0.3", "fast-glob": "^3.3.2", + "fastq": "^1.17.1", "flattie": "^1.1.1", "github-slugger": "^2.0.0", "gray-matter": "^4.0.3", diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index f3777c695..1dd86267b 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -1,12 +1,13 @@ import { promises as fs, existsSync } from 'node:fs'; import { isAbsolute } from 'node:path'; import { fileURLToPath } from 'node:url'; +import * as fastq from 'fastq'; import type { FSWatcher } from 'vite'; import xxhash from 'xxhash-wasm'; import { AstroUserError } from '../core/errors/errors.js'; import type { Logger } from '../core/logger/core.js'; import type { AstroSettings } from '../types/astro.js'; -import type { ContentEntryType } from '../types/public/content.js'; +import type { ContentEntryType, RefreshContentOptions } from '../types/public/content.js'; import { ASSET_IMPORTS_FILE, CONTENT_LAYER_TYPE, @@ -39,7 +40,8 @@ export class ContentLayer { #generateDigest?: (data: Record<string, unknown> | string) => string; - #loading = false; + #queue: fastq.queueAsPromised<RefreshContentOptions, void>; + constructor({ settings, logger, store, watcher }: ContentLayerOptions) { // The default max listeners is 10, which can be exceeded when using a lot of loaders watcher?.setMaxListeners(50); @@ -48,13 +50,14 @@ export class ContentLayer { this.#store = store; this.#settings = settings; this.#watcher = watcher; + this.#queue = fastq.promise(this.#doSync.bind(this), 1); } /** * Whether the content layer is currently loading content */ get loading() { - return this.#loading; + return !this.#queue.idle(); } /** @@ -63,11 +66,7 @@ export class ContentLayer { watchContentConfig() { this.#unsubscribe?.(); this.#unsubscribe = globalContentConfigObserver.subscribe(async (ctx) => { - if ( - !this.#loading && - ctx.status === 'loaded' && - ctx.config.digest !== this.#lastConfigDigest - ) { + if (ctx.status === 'loaded' && ctx.config.digest !== this.#lastConfigDigest) { this.sync(); } }); @@ -77,23 +76,6 @@ export class ContentLayer { this.#unsubscribe?.(); } - /** - * Run the `load()` method of each collection's loader, which will load the data and save it in the data store. - * The loader itself is responsible for deciding whether this will clear and reload the full collection, or - * perform an incremental update. After the data is loaded, the data store is written to disk. - */ - async sync() { - if (this.#loading) { - return; - } - this.#loading = true; - try { - await this.#doSync(); - } finally { - this.#loading = false; - } - } - async #getGenerateDigest() { if (this.#generateDigest) { return this.#generateDigest; @@ -114,10 +96,12 @@ export class ContentLayer { collectionName, loaderName = 'content', parseData, + refreshContextData, }: { collectionName: string; loaderName: string; parseData: LoaderContext['parseData']; + refreshContextData?: Record<string, unknown>; }): Promise<LoaderContext> { return { collection: collectionName, @@ -128,6 +112,7 @@ export class ContentLayer { parseData, generateDigest: await this.#getGenerateDigest(), watcher: this.#watcher, + refreshContextData, entryTypes: getEntryConfigByExtMap([ ...this.#settings.contentEntryTypes, ...this.#settings.dataEntryTypes, @@ -135,7 +120,18 @@ export class ContentLayer { }; } - async #doSync() { + /** + * Enqueues a sync job that runs the `load()` method of each collection's loader, which will load the data and save it in the data store. + * The loader itself is responsible for deciding whether this will clear and reload the full collection, or + * perform an incremental update. After the data is loaded, the data store is written to disk. Jobs are queued, + * so that only one sync can run at a time. The function returns a promise that resolves when this sync job is complete. + */ + + sync(options: RefreshContentOptions = {}): Promise<void> { + return this.#queue.push(options); + } + + async #doSync(options: RefreshContentOptions) { const contentConfig = globalContentConfigObserver.get(); const logger = this.#logger.forkIntegrationLogger('content'); if (contentConfig?.status !== 'loaded') { @@ -181,6 +177,15 @@ export class ContentLayer { } } + // If loaders are specified, only sync the specified loaders + if ( + options?.loaders && + (typeof collection.loader !== 'object' || + !options.loaders.includes(collection.loader.name)) + ) { + return; + } + const collectionWithResolvedSchema = { ...collection, schema }; const parseData: LoaderContext['parseData'] = async ({ id, data, filePath = '' }) => { @@ -214,6 +219,7 @@ export class ContentLayer { collectionName: name, parseData, loaderName: collection.loader.name, + refreshContextData: options?.context, }); if (typeof collection.loader === 'function') { @@ -294,18 +300,12 @@ export async function simpleLoader<TData extends { id: string }>( function contentLayerSingleton() { let instance: ContentLayer | null = null; return { - initialized: () => Boolean(instance), init: (options: ContentLayerOptions) => { instance?.unwatchContentConfig(); instance = new ContentLayer(options); return instance; }, - get: () => { - if (!instance) { - throw new Error('Content layer not initialized'); - } - return instance; - }, + get: () => instance, dispose: () => { instance?.unwatchContentConfig(); instance = null; diff --git a/packages/astro/src/content/loaders/types.ts b/packages/astro/src/content/loaders/types.ts index 1a1508207..86411407f 100644 --- a/packages/astro/src/content/loaders/types.ts +++ b/packages/astro/src/content/loaders/types.ts @@ -33,6 +33,7 @@ export interface LoaderContext { /** When running in dev, this is a filesystem watcher that can be used to trigger updates */ watcher?: FSWatcher; + refreshContextData?: Record<string, unknown>; entryTypes: Map<string, ContentEntryType>; } diff --git a/packages/astro/src/core/dev/restart.ts b/packages/astro/src/core/dev/restart.ts index 8fb6e46a3..3b4c646bf 100644 --- a/packages/astro/src/core/dev/restart.ts +++ b/packages/astro/src/core/dev/restart.ts @@ -186,9 +186,7 @@ export async function createContainerWithAutomaticRestart({ key: 's', description: 'sync content layer', action: () => { - if (globalContentLayer.initialized()) { - globalContentLayer.get().sync(); - } + globalContentLayer.get()?.sync(); }, }); } diff --git a/packages/astro/src/types/public/content.ts b/packages/astro/src/types/public/content.ts index 77c0b2f17..ccc42873c 100644 --- a/packages/astro/src/types/public/content.ts +++ b/packages/astro/src/types/public/content.ts @@ -98,6 +98,11 @@ export interface ContentEntryType { handlePropagation?: boolean; } +export interface RefreshContentOptions { + loaders?: Array<string>; + context?: Record<string, any>; +} + type GetContentEntryInfoReturnType = { data: Record<string, unknown>; /** |