diff options
| -rw-r--r-- | .changeset/curvy-walls-kneel.md | 45 | ||||
| -rw-r--r-- | packages/astro/src/content/content-layer.ts | 18 | ||||
| -rw-r--r-- | packages/astro/src/content/data-store.ts | 3 | ||||
| -rw-r--r-- | packages/astro/src/content/loaders/types.ts | 1 | ||||
| -rw-r--r-- | packages/astro/src/content/vite-plugin-content-virtual-mod.ts | 42 | ||||
| -rw-r--r-- | packages/astro/src/core/build/index.ts | 7 | ||||
| -rw-r--r-- | packages/astro/src/core/dev/dev.ts | 5 | ||||
| -rw-r--r-- | packages/astro/src/core/sync/index.ts | 16 | ||||
| -rw-r--r-- | packages/astro/src/integrations/hooks.ts | 28 | ||||
| -rw-r--r-- | packages/astro/src/types/public/integrations.ts | 2 | ||||
| -rw-r--r-- | packages/astro/test/content-layer.test.js | 26 | ||||
| -rw-r--r-- | packages/astro/test/fixtures/content-layer/astro.config.mjs | 33 | ||||
| -rw-r--r-- | packages/astro/test/test-utils.js | 22 |
13 files changed, 209 insertions, 39 deletions
diff --git a/.changeset/curvy-walls-kneel.md b/.changeset/curvy-walls-kneel.md new file mode 100644 index 000000000..dc04cb0af --- /dev/null +++ b/.changeset/curvy-walls-kneel.md @@ -0,0 +1,45 @@ +--- +'astro': patch +--- + +Adds a new function `refreshContent` to the `astro:server:setup` hook that allows integrations to refresh the content layer. This can be used, for example, to register a webhook endpoint during dev, or to open a socket to a CMS to listen for changes. + +By default, `refreshContent` will refresh all collections. You can optionally pass a `loaders` property, which is an array of loader names. If provided, only collections that use those loaders will be refreshed. For example, A CMS integration could use this property to only refresh its own collections. + +You can also pass a `context` object to the loaders. This can be used to pass arbitrary data, such as the webhook body, or an event from the websocket. + +```ts + { + name: 'my-integration', + hooks: { + 'astro:server:setup': async ({ server, refreshContent }) => { + server.middlewares.use('/_refresh', async (req, res) => { + if(req.method !== 'POST') { + res.statusCode = 405 + res.end('Method Not Allowed'); + return + } + let body = ''; + req.on('data', chunk => { + body += chunk.toString(); + }); + req.on('end', async () => { + try { + const webhookBody = JSON.parse(body); + await refreshContent({ + context: { webhookBody }, + loaders: ['my-loader'] + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ message: 'Content refreshed successfully' })); + } catch (error) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message })); + } + }); + }); + } + } +} +``` + diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index fd1aeca18..c63e055c9 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -73,6 +73,11 @@ export class ContentLayer { this.#unsubscribe?.(); } + dispose() { + this.#queue.kill(); + this.#unsubscribe?.(); + } + async #getGenerateDigest() { if (this.#generateDigest) { return this.#generateDigest; @@ -224,7 +229,7 @@ export class ContentLayer { if (!existsSync(this.#settings.config.cacheDir)) { await fs.mkdir(this.#settings.config.cacheDir, { recursive: true }); } - const cacheFile = new URL(DATA_STORE_FILE, this.#settings.config.cacheDir); + const cacheFile = getDataStoreFile(this.#settings); await this.#store.writeToDisk(cacheFile); if (!existsSync(this.#settings.dotAstroDir)) { await fs.mkdir(this.#settings.dotAstroDir, { recursive: true }); @@ -285,17 +290,24 @@ export async function simpleLoader<TData extends { id: string }>( } } +export function getDataStoreFile(settings: AstroSettings) { + return new URL( + DATA_STORE_FILE, + process?.env.NODE_ENV === 'development' ? settings.dotAstroDir : settings.config.cacheDir, + ); +} + function contentLayerSingleton() { let instance: ContentLayer | null = null; return { init: (options: ContentLayerOptions) => { - instance?.unwatchContentConfig(); + instance?.dispose(); instance = new ContentLayer(options); return instance; }, get: () => instance, dispose: () => { - instance?.unwatchContentConfig(); + instance?.dispose(); instance = null; }, }; diff --git a/packages/astro/src/content/data-store.ts b/packages/astro/src/content/data-store.ts index fbf31d0f1..21d59363c 100644 --- a/packages/astro/src/content/data-store.ts +++ b/packages/astro/src/content/data-store.ts @@ -91,6 +91,9 @@ export class DataStore { try { // @ts-expect-error - this is a virtual module const data = await import('astro:data-layer-content'); + if (data.default instanceof Map) { + return DataStore.fromMap(data.default); + } const map = devalue.unflatten(data.default); return DataStore.fromMap(map); } catch {} diff --git a/packages/astro/src/content/loaders/types.ts b/packages/astro/src/content/loaders/types.ts index 86411407f..32a3e929a 100644 --- a/packages/astro/src/content/loaders/types.ts +++ b/packages/astro/src/content/loaders/types.ts @@ -33,6 +33,7 @@ export interface LoaderContext { /** When running in dev, this is a filesystem watcher that can be used to trigger updates */ watcher?: FSWatcher; + /** If the loader has been triggered by an integration, this may optionally contain extra data set by that integration */ refreshContextData?: Record<string, unknown>; entryTypes: Map<string, ContentEntryType>; } diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index ad0b48723..a3ecfa5af 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -54,12 +54,16 @@ export function astroContentVirtualModPlugin({ }: AstroContentVirtualModPluginParams): Plugin { let IS_DEV = false; const IS_SERVER = isServerLikeOutput(settings.config); - const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir); + let dataStoreFile: URL; return { name: 'astro-content-virtual-mod-plugin', enforce: 'pre', configResolved(config) { IS_DEV = config.mode === 'development'; + dataStoreFile = new URL( + DATA_STORE_FILE, + IS_DEV ? settings.dotAstroDir : settings.config.cacheDir, + ); }, async resolveId(id) { if (id === VIRTUAL_MODULE_ID) { @@ -180,25 +184,31 @@ export function astroContentVirtualModPlugin({ configureServer(server) { const dataStorePath = fileURLToPath(dataStoreFile); - // Watch for changes to the data store file - if (Array.isArray(server.watcher.options.ignored)) { - // The data store file is in node_modules, so is ignored by default, - // so we need to un-ignore it. - server.watcher.options.ignored.push(`!${dataStorePath}`); - } + server.watcher.add(dataStorePath); + function invalidateDataStore() { + const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID); + if (module) { + server.moduleGraph.invalidateModule(module); + } + server.ws.send({ + type: 'full-reload', + path: '*', + }); + } + + // If the datastore file changes, invalidate the virtual module + + server.watcher.on('add', (addedPath) => { + if (addedPath === dataStorePath) { + invalidateDataStore(); + } + }); + server.watcher.on('change', (changedPath) => { - // If the datastore file changes, invalidate the virtual module if (changedPath === dataStorePath) { - const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID); - if (module) { - server.moduleGraph.invalidateModule(module); - } - server.ws.send({ - type: 'full-reload', - path: '*', - }); + invalidateDataStore(); } }); }, diff --git a/packages/astro/src/core/build/index.ts b/packages/astro/src/core/build/index.ts index 74a648304..10e6b2dbc 100644 --- a/packages/astro/src/core/build/index.ts +++ b/packages/astro/src/core/build/index.ts @@ -56,6 +56,9 @@ export default async function build( const logger = createNodeLogger(inlineConfig); const { userConfig, astroConfig } = await resolveConfig(inlineConfig, 'build'); telemetry.record(eventCliSession('build', userConfig)); + + const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root)); + if (inlineConfig.force) { if (astroConfig.experimental.contentCollectionCache) { const contentCacheDir = new URL('./content/', astroConfig.cacheDir); @@ -65,11 +68,9 @@ export default async function build( logger.warn('content', 'content cache cleared (force)'); } } - await clearContentLayerCache({ astroConfig, logger, fs }); + await clearContentLayerCache({ settings, logger, fs }); } - const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root)); - const builder = new AstroBuilder(settings, { ...options, logger, diff --git a/packages/astro/src/core/dev/dev.ts b/packages/astro/src/core/dev/dev.ts index 73ec0fa71..72a173dbe 100644 --- a/packages/astro/src/core/dev/dev.ts +++ b/packages/astro/src/core/dev/dev.ts @@ -5,8 +5,7 @@ import { performance } from 'node:perf_hooks'; import { green } from 'kleur/colors'; import { gt, major, minor, patch } from 'semver'; import type * as vite from 'vite'; -import { DATA_STORE_FILE } from '../../content/consts.js'; -import { globalContentLayer } from '../../content/content-layer.js'; +import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js'; import { attachContentServerListeners } from '../../content/index.js'; import { MutableDataStore } from '../../content/mutable-data-store.js'; import { globalContentConfigObserver } from '../../content/utils.js'; @@ -108,7 +107,7 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise<DevS let store: MutableDataStore | undefined; try { - const dataStoreFile = new URL(DATA_STORE_FILE, restart.container.settings.config.cacheDir); + const dataStoreFile = getDataStoreFile(restart.container.settings); if (existsSync(dataStoreFile)) { store = await MutableDataStore.fromFile(dataStoreFile); } diff --git a/packages/astro/src/core/sync/index.ts b/packages/astro/src/core/sync/index.ts index 9b2973233..88c37fa88 100644 --- a/packages/astro/src/core/sync/index.ts +++ b/packages/astro/src/core/sync/index.ts @@ -3,8 +3,8 @@ import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; import { dim } from 'kleur/colors'; import { type HMRPayload, createServer } from 'vite'; -import { CONTENT_TYPES_FILE, DATA_STORE_FILE } from '../../content/consts.js'; -import { globalContentLayer } from '../../content/content-layer.js'; +import { CONTENT_TYPES_FILE } from '../../content/consts.js'; +import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js'; import { createContentTypesGenerator } from '../../content/index.js'; import { MutableDataStore } from '../../content/mutable-data-store.js'; import { getContentPaths, globalContentConfigObserver } from '../../content/utils.js'; @@ -13,7 +13,7 @@ import { telemetry } from '../../events/index.js'; import { eventCliSession } from '../../events/session.js'; import { runHookConfigDone, runHookConfigSetup } from '../../integrations/hooks.js'; import type { AstroSettings } from '../../types/astro.js'; -import type { AstroConfig, AstroInlineConfig } from '../../types/public/config.js'; +import type { AstroInlineConfig } from '../../types/public/config.js'; import { getTimeStat } from '../build/util.js'; import { resolveConfig } from '../config/config.js'; import { createNodeLogger } from '../config/logging.js'; @@ -70,11 +70,11 @@ export default async function sync( * Clears the content layer and content collection cache, forcing a full rebuild. */ export async function clearContentLayerCache({ - astroConfig, + settings, logger, fs = fsMod, -}: { astroConfig: AstroConfig; logger: Logger; fs?: typeof fsMod }) { - const dataStore = new URL(DATA_STORE_FILE, astroConfig.cacheDir); +}: { settings: AstroSettings; logger: Logger; fs?: typeof fsMod }) { + const dataStore = getDataStoreFile(settings); if (fs.existsSync(dataStore)) { logger.debug('content', 'clearing data store'); await fs.promises.rm(dataStore, { force: true }); @@ -96,7 +96,7 @@ export async function syncInternal({ force, }: SyncOptions): Promise<void> { if (force) { - await clearContentLayerCache({ astroConfig: settings.config, logger, fs }); + await clearContentLayerCache({ settings, logger, fs }); } const timerStart = performance.now(); @@ -107,7 +107,7 @@ export async function syncInternal({ settings.timer.start('Sync content layer'); let store: MutableDataStore | undefined; try { - const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir); + const dataStoreFile = getDataStoreFile(settings); if (existsSync(dataStoreFile)) { store = await MutableDataStore.fromFile(dataStoreFile); } diff --git a/packages/astro/src/integrations/hooks.ts b/packages/astro/src/integrations/hooks.ts index b86281cbb..1e4ad5ca3 100644 --- a/packages/astro/src/integrations/hooks.ts +++ b/packages/astro/src/integrations/hooks.ts @@ -5,6 +5,9 @@ import { bold } from 'kleur/colors'; import type { InlineConfig, ViteDevServer } from 'vite'; import astroIntegrationActionsRouteHandler from '../actions/integration.js'; import { isActionsFilePresent } from '../actions/utils.js'; +import { CONTENT_LAYER_TYPE } from '../content/consts.js'; +import { globalContentLayer } from '../content/content-layer.js'; +import { globalContentConfigObserver } from '../content/utils.js'; import type { SerializedSSRManifest } from '../core/app/types.js'; import type { PageBuildData } from '../core/build/types.js'; import { buildClientDirectiveEntrypoint } from '../core/client-directive/index.js'; @@ -13,7 +16,11 @@ import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js'; import { isServerLikeOutput } from '../core/util.js'; import type { AstroSettings } from '../types/astro.js'; import type { AstroConfig } from '../types/public/config.js'; -import type { ContentEntryType, DataEntryType } from '../types/public/content.js'; +import type { + ContentEntryType, + DataEntryType, + RefreshContentOptions, +} from '../types/public/content.js'; import type { AstroIntegration, AstroRenderer, @@ -367,6 +374,24 @@ export async function runHookServerSetup({ server: ViteDevServer; logger: Logger; }) { + let refreshContent: undefined | ((options: RefreshContentOptions) => Promise<void>); + if (config.experimental?.contentLayer) { + refreshContent = async (options: RefreshContentOptions) => { + const contentConfig = globalContentConfigObserver.get(); + if ( + contentConfig.status !== 'loaded' || + !Object.values(contentConfig.config.collections).some( + (collection) => collection.type === CONTENT_LAYER_TYPE, + ) + ) { + return; + } + + const contentLayer = await globalContentLayer.get(); + await contentLayer?.sync(options); + }; + } + for (const integration of config.integrations) { if (integration?.hooks?.['astro:server:setup']) { await withTakingALongTimeMsg({ @@ -376,6 +401,7 @@ export async function runHookServerSetup({ server, logger: getLogger(integration, logger), toolbar: getToolbarServerCommunicationHelpers(server), + refreshContent, }), logger, }); diff --git a/packages/astro/src/types/public/integrations.ts b/packages/astro/src/types/public/integrations.ts index a65f2513b..19cc1e298 100644 --- a/packages/astro/src/types/public/integrations.ts +++ b/packages/astro/src/types/public/integrations.ts @@ -6,6 +6,7 @@ import type { AstroIntegrationLogger } from '../../core/logger/core.js'; import type { getToolbarServerCommunicationHelpers } from '../../integrations/hooks.js'; import type { DeepPartial } from '../../type-utils.js'; import type { AstroConfig } from './config.js'; +import type { RefreshContentOptions } from './content.js'; import type { RouteData } from './internal.js'; import type { DevToolbarAppEntry } from './toolbar.js'; @@ -187,6 +188,7 @@ export interface BaseIntegrationHooks { server: ViteDevServer; logger: AstroIntegrationLogger; toolbar: ReturnType<typeof getToolbarServerCommunicationHelpers>; + refreshContent?: (options: RefreshContentOptions) => Promise<void>; }) => void | Promise<void>; 'astro:server:start': (options: { address: AddressInfo; diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 0590e7e59..75d702a94 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -196,7 +196,11 @@ describe('Content Layer', () => { let devServer; let json; before(async () => { - devServer = await fixture.startDevServer(); + devServer = await fixture.startDevServer({ force: true }); + // Vite may not have noticed the saved data store yet. Wait a little just in case. + await fixture.onNextDataStoreChange(1000).catch(() => { + // Ignore timeout, because it may have saved before we get here. + }) const rawJsonResponse = await fixture.fetch('/collections.json'); const rawJson = await rawJsonResponse.text(); json = devalue.parse(rawJson); @@ -275,6 +279,22 @@ describe('Content Layer', () => { }); }); + it('reloads data when an integration triggers a content refresh', async () => { + const rawJsonResponse = await fixture.fetch('/collections.json'); + const initialJson = devalue.parse(await rawJsonResponse.text()); + assert.equal(initialJson.increment.data.lastValue, 1); + + const refreshResponse = await fixture.fetch('/_refresh', { + method: 'POST', + body: JSON.stringify({}), + }); + const refreshData = await refreshResponse.json(); + assert.equal(refreshData.message, 'Content refreshed successfully'); + const updatedJsonResponse = await fixture.fetch('/collections.json'); + const updated = devalue.parse(await updatedJsonResponse.text()); + assert.equal(updated.increment.data.lastValue, 2); + }); + it('updates collection when data file is changed', async () => { const rawJsonResponse = await fixture.fetch('/collections.json'); const initialJson = devalue.parse(await rawJsonResponse.text()); @@ -286,9 +306,7 @@ describe('Content Layer', () => { return JSON.stringify(data, null, 2); }); - // Writes are debounced to 500ms - await new Promise((r) => setTimeout(r, 700)); - + await fixture.onNextDataStoreChange(); const updatedJsonResponse = await fixture.fetch('/collections.json'); const updated = devalue.parse(await updatedJsonResponse.text()); assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy')); diff --git a/packages/astro/test/fixtures/content-layer/astro.config.mjs b/packages/astro/test/fixtures/content-layer/astro.config.mjs index 3266e5e8c..37afc3992 100644 --- a/packages/astro/test/fixtures/content-layer/astro.config.mjs +++ b/packages/astro/test/fixtures/content-layer/astro.config.mjs @@ -3,7 +3,38 @@ import { defineConfig } from 'astro/config'; import { fileURLToPath } from 'node:url'; export default defineConfig({ - integrations: [mdx()], + integrations: [mdx(), { + name: '@astrojs/my-integration', + hooks: { + 'astro:server:setup': async ({ server, refreshContent }) => { + server.middlewares.use('/_refresh', async (req, res) => { + if(req.method !== 'POST') { + res.statusCode = 405 + res.end('Method Not Allowed'); + return + } + let body = ''; + req.on('data', chunk => { + body += chunk.toString(); + }); + req.on('end', async () => { + try { + const webhookBody = JSON.parse(body); + await refreshContent({ + context: { webhookBody }, + loaders: ['increment-loader'] + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ message: 'Content refreshed successfully' })); + } catch (error) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message })); + } + }); + }); + } + } +}], vite: { resolve: { alias: { diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index facaaef00..00a8099f4 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -45,6 +45,7 @@ process.env.ASTRO_TELEMETRY_DISABLED = true; * @property {() => Promise<App>} loadTestAdapterApp * @property {() => Promise<(req: NodeRequest, res: NodeResponse) => void>} loadNodeAdapterHandler * @property {() => Promise<void>} onNextChange + * @property {(timeout?: number) => Promise<void>} onNextDataStoreChange * @property {typeof check} check * @property {typeof sync} sync * @property {AstroConfig} config @@ -180,6 +181,27 @@ export async function loadFixture(inlineConfig) { config.server.port = devServer.address.port; // update port return devServer; }, + onNextDataStoreChange: (timeout = 5000) => { + if (!devServer) { + return Promise.reject(new Error('No dev server running')); + } + + const dataStoreFile = path.join(root, '.astro', 'data-store.json'); + + return new Promise((resolve, reject) => { + const changeHandler = (fileName) => { + if (fileName === dataStoreFile) { + devServer.watcher.removeListener('change', changeHandler); + resolve(); + } + }; + devServer.watcher.on('change', changeHandler); + setTimeout(() => { + devServer.watcher.removeListener('change', changeHandler); + reject(new Error('Data store did not update within timeout')); + }, timeout); + }); + }, config, resolveUrl, fetch: async (url, init) => { |