Make vite-plugin-content-virtual-mod run getEntrySlug 10 at a time (#…

…7125)
withastro · May 23, 2023 · 4ce8bf7 · 4ce8bf7
1 parent 2ca9426
commit 4ce8bf7
Show file tree

Hide file tree

Showing 4 changed files with 71 additions and 52 deletions.
diff --git a/.changeset/dull-teachers-work.md b/.changeset/dull-teachers-work.md
@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Make vite-plugin-content-virtual-mod run `getEntrySlug` 10 at a time to prevent `EMFILE: too many open files` error
diff --git a/packages/astro/package.json b/packages/astro/package.json
@@ -148,6 +148,7 @@
     "magic-string": "^0.27.0",
     "mime": "^3.0.0",
     "ora": "^6.1.0",
+    "p-limit": "^4.0.0",
     "path-to-regexp": "^6.2.1",
     "preferred-pm": "^3.0.3",
     "prompts": "^2.4.2",

diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts
@@ -1,4 +1,5 @@
 import glob from 'fast-glob';
+import pLimit from 'p-limit';
 import fsMod from 'node:fs';
 import { extname } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -114,59 +115,68 @@ export async function getStringifiedLookupMap({
 		}
 	);
 
-	await Promise.all(
-		contentGlob.map(async (filePath) => {
-			const entryType = getEntryType(filePath, contentPaths, contentEntryExts, dataEntryExts);
-			// Globbed ignored or unsupported entry.
-			// Logs warning during type generation, should ignore in lookup map.
-			if (entryType !== 'content' && entryType !== 'data') return;
-
-			const collection = getEntryCollectionName({ contentDir, entry: pathToFileURL(filePath) });
-			if (!collection) throw UnexpectedLookupMapError;
-
-			if (lookupMap[collection]?.type && lookupMap[collection].type !== entryType) {
-				throw new AstroError({
-					...AstroErrorData.MixedContentDataCollectionError,
-					message: AstroErrorData.MixedContentDataCollectionError.message(collection),
-				});
-			}
-
-			if (entryType === 'content') {
-				const contentEntryType = contentEntryConfigByExt.get(extname(filePath));
-				if (!contentEntryType) throw UnexpectedLookupMapError;
+	// Run 10 at a time to prevent `await getEntrySlug` from accessing the filesystem all at once.
+	// Each await shouldn't take too long for the work to be noticably slow too.
+	const limit = pLimit(10);
+	const promises: Promise<void>[] = [];
+
+	for (const filePath of contentGlob) {
+		promises.push(
+			limit(async () => {
+				const entryType = getEntryType(filePath, contentPaths, contentEntryExts, dataEntryExts);
+				// Globbed ignored or unsupported entry.
+				// Logs warning during type generation, should ignore in lookup map.
+				if (entryType !== 'content' && entryType !== 'data') return;
+
+				const collection = getEntryCollectionName({ contentDir, entry: pathToFileURL(filePath) });
+				if (!collection) throw UnexpectedLookupMapError;
+
+				if (lookupMap[collection]?.type && lookupMap[collection].type !== entryType) {
+					throw new AstroError({
+						...AstroErrorData.MixedContentDataCollectionError,
+						message: AstroErrorData.MixedContentDataCollectionError.message(collection),
+					});
+				}
+
+				if (entryType === 'content') {
+					const contentEntryType = contentEntryConfigByExt.get(extname(filePath));
+					if (!contentEntryType) throw UnexpectedLookupMapError;
+
+					const { id, slug: generatedSlug } = await getContentEntryIdAndSlug({
+						entry: pathToFileURL(filePath),
+						contentDir,
+						collection,
+					});
+					const slug = await getEntrySlug({
+						id,
+						collection,
+						generatedSlug,
+						fs,
+						fileUrl: pathToFileURL(filePath),
+						contentEntryType,
+					});
+					lookupMap[collection] = {
+						type: 'content',
+						entries: {
+							...lookupMap[collection]?.entries,
+							[slug]: rootRelativePath(root, filePath),
+						},
+					};
+				} else {
+					const id = getDataEntryId({ entry: pathToFileURL(filePath), contentDir, collection });
+					lookupMap[collection] = {
+						type: 'data',
+						entries: {
+							...lookupMap[collection]?.entries,
+							[id]: rootRelativePath(root, filePath),
+						},
+					};
+				}
+			})
+		);
+	}
 
-				const { id, slug: generatedSlug } = await getContentEntryIdAndSlug({
-					entry: pathToFileURL(filePath),
-					contentDir,
-					collection,
-				});
-				const slug = await getEntrySlug({
-					id,
-					collection,
-					generatedSlug,
-					fs,
-					fileUrl: pathToFileURL(filePath),
-					contentEntryType,
-				});
-				lookupMap[collection] = {
-					type: 'content',
-					entries: {
-						...lookupMap[collection]?.entries,
-						[slug]: rootRelativePath(root, filePath),
-					},
-				};
-			} else {
-				const id = getDataEntryId({ entry: pathToFileURL(filePath), contentDir, collection });
-				lookupMap[collection] = {
-					type: 'data',
-					entries: {
-						...lookupMap[collection]?.entries,
-						[id]: rootRelativePath(root, filePath),
-					},
-				};
-			}
-		})
-	);
+	await Promise.all(promises);
 
 	return JSON.stringify(lookupMap);
 }

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml