diff options
| -rw-r--r-- | .changeset/dull-teachers-work.md | 5 | ||||
| -rw-r--r-- | packages/astro/package.json | 1 | ||||
| -rw-r--r-- | packages/astro/src/content/vite-plugin-content-virtual-mod.ts | 114 | ||||
| -rw-r--r-- | pnpm-lock.yaml | 3 |
4 files changed, 71 insertions, 52 deletions
diff --git a/.changeset/dull-teachers-work.md b/.changeset/dull-teachers-work.md new file mode 100644 index 000000000..a0a3ca5e3 --- /dev/null +++ b/.changeset/dull-teachers-work.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Make vite-plugin-content-virtual-mod run `getEntrySlug` 10 at a time to prevent `EMFILE: too many open files` error diff --git a/packages/astro/package.json b/packages/astro/package.json index 5094ec543..8fa45a077 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -148,6 +148,7 @@ "magic-string": "^0.27.0", "mime": "^3.0.0", "ora": "^6.1.0", + "p-limit": "^4.0.0", "path-to-regexp": "^6.2.1", "preferred-pm": "^3.0.3", "prompts": "^2.4.2", diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index 51ac99315..ef1601ff9 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -1,4 +1,5 @@ import glob from 'fast-glob'; +import pLimit from 'p-limit'; import fsMod from 'node:fs'; import { extname } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -114,59 +115,68 @@ export async function getStringifiedLookupMap({ } ); - await Promise.all( - contentGlob.map(async (filePath) => { - const entryType = getEntryType(filePath, contentPaths, contentEntryExts, dataEntryExts); - // Globbed ignored or unsupported entry. - // Logs warning during type generation, should ignore in lookup map. - if (entryType !== 'content' && entryType !== 'data') return; - - const collection = getEntryCollectionName({ contentDir, entry: pathToFileURL(filePath) }); - if (!collection) throw UnexpectedLookupMapError; - - if (lookupMap[collection]?.type && lookupMap[collection].type !== entryType) { - throw new AstroError({ - ...AstroErrorData.MixedContentDataCollectionError, - message: AstroErrorData.MixedContentDataCollectionError.message(collection), - }); - } - - if (entryType === 'content') { - const contentEntryType = contentEntryConfigByExt.get(extname(filePath)); - if (!contentEntryType) throw UnexpectedLookupMapError; + // Run 10 at a time to prevent `await getEntrySlug` from accessing the filesystem all at once. + // Each await shouldn't take too long for the work to be noticably slow too. + const limit = pLimit(10); + const promises: Promise<void>[] = []; + + for (const filePath of contentGlob) { + promises.push( + limit(async () => { + const entryType = getEntryType(filePath, contentPaths, contentEntryExts, dataEntryExts); + // Globbed ignored or unsupported entry. + // Logs warning during type generation, should ignore in lookup map. + if (entryType !== 'content' && entryType !== 'data') return; + + const collection = getEntryCollectionName({ contentDir, entry: pathToFileURL(filePath) }); + if (!collection) throw UnexpectedLookupMapError; + + if (lookupMap[collection]?.type && lookupMap[collection].type !== entryType) { + throw new AstroError({ + ...AstroErrorData.MixedContentDataCollectionError, + message: AstroErrorData.MixedContentDataCollectionError.message(collection), + }); + } + + if (entryType === 'content') { + const contentEntryType = contentEntryConfigByExt.get(extname(filePath)); + if (!contentEntryType) throw UnexpectedLookupMapError; + + const { id, slug: generatedSlug } = await getContentEntryIdAndSlug({ + entry: pathToFileURL(filePath), + contentDir, + collection, + }); + const slug = await getEntrySlug({ + id, + collection, + generatedSlug, + fs, + fileUrl: pathToFileURL(filePath), + contentEntryType, + }); + lookupMap[collection] = { + type: 'content', + entries: { + ...lookupMap[collection]?.entries, + [slug]: rootRelativePath(root, filePath), + }, + }; + } else { + const id = getDataEntryId({ entry: pathToFileURL(filePath), contentDir, collection }); + lookupMap[collection] = { + type: 'data', + entries: { + ...lookupMap[collection]?.entries, + [id]: rootRelativePath(root, filePath), + }, + }; + } + }) + ); + } - const { id, slug: generatedSlug } = await getContentEntryIdAndSlug({ - entry: pathToFileURL(filePath), - contentDir, - collection, - }); - const slug = await getEntrySlug({ - id, - collection, - generatedSlug, - fs, - fileUrl: pathToFileURL(filePath), - contentEntryType, - }); - lookupMap[collection] = { - type: 'content', - entries: { - ...lookupMap[collection]?.entries, - [slug]: rootRelativePath(root, filePath), - }, - }; - } else { - const id = getDataEntryId({ entry: pathToFileURL(filePath), contentDir, collection }); - lookupMap[collection] = { - type: 'data', - entries: { - ...lookupMap[collection]?.entries, - [id]: rootRelativePath(root, filePath), - }, - }; - } - }) - ); + await Promise.all(promises); return JSON.stringify(lookupMap); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20ca7bcb5..df4ad8e2b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -639,6 +639,9 @@ importers: ora: specifier: ^6.1.0 version: 6.1.0 + p-limit: + specifier: ^4.0.0 + version: 4.0.0 path-to-regexp: specifier: ^6.2.1 version: 6.2.1 |