fix(astro): handle symlinked content collection directories (#11236)

* fix(astro): handle symlinked content collection directories

* CHeck content dir exists and is a dir

* Handle symlinks when generating chunk names

* wip windows log

* Use posix paths

* Fix normalisation

* :old-man-yells-at-windows-paths:

* Update .changeset/fifty-clouds-clean.md

Co-authored-by: Emanuele Stoppa <my.burning@gmail.com>

* Changes from review

* Add logging

---------

Co-authored-by: Emanuele Stoppa <my.burning@gmail.com>

Author: ascorbic

.changeset/fifty-clouds-clean.md

@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Fixes a case where symlinked content collection directories were not correctly resolved

packages/astro/src/content/utils.ts

@@ -16,7 +16,7 @@ import { AstroError, AstroErrorData, MarkdownError, errorMap } from '../core/err
 import { isYAMLException } from '../core/errors/utils.js';
 import { CONTENT_FLAGS, PROPAGATED_ASSET_FLAG } from './consts.js';
 import { createImage } from './runtime-assets.js';
-
+import type { Logger } from "../core/logger/core.js";
 /**
  * Amap from a collection + slug to the local file path.
  * This is used internally to resolve entry imports when using `getEntry()`.
@@ -167,6 +167,67 @@ export function getEntryConfigByExtMap<TEntryType extends ContentEntryType | Dat
 	return map;
 }
 
+export async function getSymlinkedContentCollections({
+	contentDir,
+	logger,
+	fs
+}: {
+	contentDir: URL;
+	logger: Logger;
+	fs: typeof fsMod;
+}): Promise<Map<string, string>> {
+	const contentPaths = new Map<string, string>();
+	const contentDirPath = fileURLToPath(contentDir);
+	try {
+		if (!fs.existsSync(contentDirPath) || !fs.lstatSync(contentDirPath).isDirectory()) {
+			return contentPaths;
+		}
+	} catch {
+		// Ignore if there isn't a valid content directory
+		return contentPaths;
+	}
+	try {
+		const contentDirEntries = await fs.promises.readdir(contentDir, { withFileTypes: true });
+		for (const entry of contentDirEntries) {
+			if (entry.isSymbolicLink()) {
+				const entryPath = path.join(contentDirPath, entry.name);
+				const realPath = await fs.promises.realpath(entryPath);
+				contentPaths.set(normalizePath(realPath), entry.name);
+			}
+		}
+	} catch (e) {
+		logger.warn('content', `Error when reading content directory "${contentDir}"`);
+		logger.debug('content', e);
+		// If there's an error, return an empty map
+		return new Map<string, string>();
+	}
+
+	return contentPaths;
+}
+
+export function reverseSymlink({
+	entry,
+	symlinks,
+	contentDir,
+}: {
+	entry: string | URL;
+	contentDir: string | URL;
+	symlinks?: Map<string, string>;
+}): string {
+	const entryPath = normalizePath(typeof entry === 'string' ? entry : fileURLToPath(entry));
+	const contentDirPath = typeof contentDir === 'string' ? contentDir : fileURLToPath(contentDir);
+	if (!symlinks || symlinks.size === 0) {
+		return entryPath;
+	}
+
+	for (const [realPath, symlinkName] of symlinks) {
+		if (entryPath.startsWith(realPath)) {
+			return normalizePath(path.join(contentDirPath, symlinkName, entryPath.replace(realPath, '')));
+		}
+	}
+	return entryPath;
+}
+
 export function getEntryCollectionName({
 	contentDir,
 	entry,

packages/astro/src/content/vite-plugin-content-imports.ts

@@ -28,11 +28,14 @@ import {
 	getEntryConfigByExtMap,
 	getEntryData,
 	getEntryType,
+	getSymlinkedContentCollections,
 	globalContentConfigObserver,
 	hasContentFlag,
 	parseEntrySlug,
 	reloadContentConfigObserver,
+	reverseSymlink,
 } from './utils.js';
+import type { Logger } from '../core/logger/core.js';
 
 function getContentRendererByViteId(
 	viteId: string,
@@ -63,9 +66,11 @@ const COLLECTION_TYPES_TO_INVALIDATE_ON = ['data', 'content', 'config'];
 export function astroContentImportPlugin({
 	fs,
 	settings,
+	logger,
 }: {
 	fs: typeof fsMod;
 	settings: AstroSettings;
+	logger: Logger;
 }): Plugin[] {
 	const contentPaths = getContentPaths(settings.config, fs);
 	const contentEntryExts = getContentEntryExts(settings);
@@ -75,16 +80,26 @@ export function astroContentImportPlugin({
 	const dataEntryConfigByExt = getEntryConfigByExtMap(settings.dataEntryTypes);
 	const { contentDir } = contentPaths;
 	let shouldEmitFile = false;
-
+	let symlinks: Map<string, string>;
 	const plugins: Plugin[] = [
 		{
 			name: 'astro:content-imports',
 			config(_config, env) {
 				shouldEmitFile = env.command === 'build';
 			},
+			async buildStart() {
+				// Get symlinks once at build start
+				symlinks = await getSymlinkedContentCollections({ contentDir, logger, fs });
+			},
 			async transform(_, viteId) {
 				if (hasContentFlag(viteId, DATA_FLAG)) {
-					const fileId = viteId.split('?')[0] ?? viteId;
+					// By default, Vite will resolve symlinks to their targets. We need to reverse this for
+					// content entries, so we can get the path relative to the content directory.
+					const fileId = reverseSymlink({
+						entry: viteId.split('?')[0] ?? viteId,
+						contentDir,
+						symlinks,
+					});
 					// Data collections don't need to rely on the module cache.
 					// This cache only exists for the `render()` function specific to content.
 					const { id, data, collection, _internal } = await getDataEntryModule({
@@ -109,7 +124,7 @@ export const _internal = {
 `;
 					return code;
 				} else if (hasContentFlag(viteId, CONTENT_FLAG)) {
-					const fileId = viteId.split('?')[0];
+					const fileId = reverseSymlink({ entry: viteId.split('?')[0], contentDir, symlinks });
 					const { id, slug, collection, body, data, _internal } = await getContentEntryModule({
 						fileId,
 						entryConfigByExt: contentEntryConfigByExt,

packages/astro/src/core/build/static-build.ts

@@ -8,7 +8,11 @@ import { bgGreen, bgMagenta, black, green } from 'kleur/colors';
 import * as vite from 'vite';
 import type { RouteData } from '../../@types/astro.js';
 import { PROPAGATED_ASSET_FLAG } from '../../content/consts.js';
-import { hasAnyContentFlag } from '../../content/utils.js';
+import {
+	getSymlinkedContentCollections,
+	hasAnyContentFlag,
+	reverseSymlink,
+} from '../../content/utils.js';
 import {
 	type BuildInternals,
 	createBuildInternals,
@@ -36,9 +40,10 @@ import { RESOLVED_SPLIT_MODULE_ID, RESOLVED_SSR_VIRTUAL_MODULE_ID } from './plug
 import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js';
 import type { StaticBuildOptions } from './types.js';
 import { encodeName, getTimeStat, viteBuildReturnToRollupOutputs } from './util.js';
+import type { Logger } from '../logger/core.js';
 
 export async function viteBuild(opts: StaticBuildOptions) {
-	const { allPages, settings } = opts;
+	const { allPages, settings, logger } = opts;
 	// Make sure we have an adapter before building
 	if (isModeServerWithNoAdapter(opts.settings)) {
 		throw new AstroError(AstroErrorData.NoAdapterInstalled);
@@ -78,7 +83,7 @@ export async function viteBuild(opts: StaticBuildOptions) {
 	// Build your project (SSR application code, assets, client JS, etc.)
 	const ssrTime = performance.now();
 	opts.logger.info('build', `Building ${settings.config.output} entrypoints...`);
-	const ssrOutput = await ssrBuild(opts, internals, pageInput, container);
+	const ssrOutput = await ssrBuild(opts, internals, pageInput, container, logger);
 	opts.logger.info('build', green(`✓ Completed in ${getTimeStat(ssrTime, performance.now())}.`));
 
 	settings.timer.end('SSR build');
@@ -166,7 +171,8 @@ async function ssrBuild(
 	opts: StaticBuildOptions,
 	internals: BuildInternals,
 	input: Set<string>,
-	container: AstroBuildPluginContainer
+	container: AstroBuildPluginContainer,
+	logger: Logger
 ) {
 	const buildID = Date.now().toString();
 	const { allPages, settings, viteConfig } = opts;
@@ -175,7 +181,8 @@ async function ssrBuild(
 	const routes = Object.values(allPages).flatMap((pageData) => pageData.route);
 	const isContentCache = !ssr && settings.config.experimental.contentCollectionCache;
 	const { lastVitePlugins, vitePlugins } = await container.runBeforeHook('server', input);
-
+	const contentDir = new URL('./src/content', settings.config.root);
+	const symlinks = await getSymlinkedContentCollections({ contentDir, logger, fs });
 	const viteBuildConfig: vite.InlineConfig = {
 		...viteConfig,
 		mode: viteConfig.mode || 'production',
@@ -251,7 +258,12 @@ async function ssrBuild(
 							chunkInfo.facadeModuleId &&
 							hasAnyContentFlag(chunkInfo.facadeModuleId)
 						) {
-							const [srcRelative, flag] = chunkInfo.facadeModuleId.split('/src/')[1].split('?');
+							const moduleId = reverseSymlink({
+								symlinks,
+								entry: chunkInfo.facadeModuleId,
+								contentDir,
+							});
+							const [srcRelative, flag] = moduleId.split('/src/')[1].split('?');
 							if (flag === PROPAGATED_ASSET_FLAG) {
 								return encodeName(`${removeFileExtension(srcRelative)}.entry.mjs`);
 							}

packages/astro/src/core/create-vite.ts

@@ -148,7 +148,7 @@ export async function createVite(
 			astroScannerPlugin({ settings, logger }),
 			astroInjectEnvTsPlugin({ settings, logger, fs }),
 			astroContentVirtualModPlugin({ fs, settings }),
-			astroContentImportPlugin({ fs, settings }),
+			astroContentImportPlugin({ fs, settings, logger }),
 			astroContentAssetPropagationPlugin({ mode, settings }),
 			vitePluginMiddleware({ settings }),
 			vitePluginSSRManifest(),

packages/astro/test/content-collections.test.js

@@ -98,6 +98,28 @@ describe('Content Collections', () => {
 					subject: 'My Newsletter',
 				});
 			});
+
+			it('Handles symlinked content', async () => {
+				assert.ok(json.hasOwnProperty('withSymlinkedContent'));
+				assert.equal(Array.isArray(json.withSymlinkedContent), true);
+
+				const ids = json.withSymlinkedContent.map((item) => item.id);
+				assert.deepEqual(ids, ['first.md', 'second.md', 'third.md']);
+				assert.equal(json.withSymlinkedContent[0].data.title, 'First Blog');
+			});
+
+			it('Handles symlinked data', async () => {
+				assert.ok(json.hasOwnProperty('withSymlinkedData'));
+				assert.equal(Array.isArray(json.withSymlinkedData), true);
+
+				const ids = json.withSymlinkedData.map((item) => item.id);
+				assert.deepEqual(ids, ['welcome']);
+				assert.equal(
+					json.withSymlinkedData[0].data.alt,
+					'Futuristic landscape with chrome buildings and blue skies'
+				);
+				assert.notEqual(json.withSymlinkedData[0].data.src.src, undefined);
+			});
 		});
 
 		describe('Propagation', () => {

packages/astro/test/fixtures/content-collections/src/assets/the-future.jpg

@@ -11,7 +11,7 @@ const withSchemaConfig = defineCollection({
 		isDraft: z.boolean().default(false),
 		lang: z.enum(['en', 'fr', 'es']).default('en'),
 		publishedAt: z.date().transform((val) => new Date(val)),
-	})
+	}),
 });
 
 const withUnionSchema = defineCollection({
@@ -28,8 +28,27 @@ const withUnionSchema = defineCollection({
 	]),
 });
 
+const withSymlinkedData = defineCollection({
+	type: 'data',
+	schema: ({ image }) =>
+		z.object({
+			alt: z.string(),
+			src: image(),
+		}),
+});
+
+const withSymlinkedContent = defineCollection({
+	type: 'content',
+	schema: z.object({
+		title: z.string(),
+		date: z.date(),
+	}),
+});
+
 export const collections = {
 	'with-custom-slugs': withCustomSlugs,
 	'with-schema-config': withSchemaConfig,
 	'with-union-schema': withUnionSchema,
-}
+	'with-symlinked-data': withSymlinkedData,
+	'with-symlinked-content': withSymlinkedContent,
+};

packages/astro/test/fixtures/content-collections/src/content/config.ts

@@ -0,0 +1 @@
+../../symlinked-collections/content-collection

packages/astro/test/fixtures/content-collections/src/content/with-symlinked-content

@@ -0,0 +1 @@
+../../symlinked-collections/data-collection

packages/astro/test/fixtures/content-collections/src/content/with-symlinked-data

@@ -7,8 +7,10 @@ export async function GET() {
 	const withSchemaConfig = stripAllRenderFn(await getCollection('with-schema-config'));
 	const withSlugConfig = stripAllRenderFn(await getCollection('with-custom-slugs'));
 	const withUnionSchema = stripAllRenderFn(await getCollection('with-union-schema'));
+	const withSymlinkedContent = stripAllRenderFn(await getCollection('with-symlinked-content'));
+	const withSymlinkedData = stripAllRenderFn(await getCollection('with-symlinked-data'));
 
 	return new Response(
-		devalue.stringify({ withoutConfig, withSchemaConfig, withSlugConfig, withUnionSchema })
+		devalue.stringify({ withoutConfig, withSchemaConfig, withSlugConfig, withUnionSchema, withSymlinkedContent, withSymlinkedData }),
 	);
 }

packages/astro/test/fixtures/content-collections/src/pages/collections.json.js

@@ -0,0 +1,6 @@
+---
+title: "First Blog"
+date: 2024-04-05
+---
+
+First blog content.

packages/astro/test/fixtures/content-collections/symlinked-collections/content-collection/first.md

@@ -0,0 +1,6 @@
+---
+title: "Second Blog"
+date: 2024-04-06
+---
+
+Second blog content.

packages/astro/test/fixtures/content-collections/symlinked-collections/content-collection/second.md

@@ -0,0 +1,6 @@
+---
+title: "Third Blog"
+date: 2024-04-07
+---
+
+Third blog content.

packages/astro/test/fixtures/content-collections/symlinked-collections/content-collection/third.md

@@ -0,0 +1,4 @@
+{
+  "alt": "Futuristic landscape with chrome buildings and blue skies",
+  "src": "../../assets/the-future.jpg"
+}