Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Preserve content modules properly in cache #10889

Merged
merged 7 commits into from May 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/spicy-keys-own.md
@@ -0,0 +1,5 @@
---
"astro": patch
---

Preserve content modules properly in cache
2 changes: 2 additions & 0 deletions packages/astro/content-module.template.mjs
Expand Up @@ -48,10 +48,12 @@ const collectionToRenderEntryMap = createCollectionToGlobResultMap({
contentDir,
});

const cacheEntriesByCollection = new Map();
export const getCollection = createGetCollection({
contentCollectionToEntryMap,
dataCollectionToEntryMap,
getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap),
cacheEntriesByCollection,
});

export const getEntryBySlug = createGetEntryBySlug({
Expand Down
2 changes: 1 addition & 1 deletion packages/astro/src/assets/build/generate.ts
Expand Up @@ -10,7 +10,7 @@ import { AstroError } from '../../core/errors/errors.js';
import { AstroErrorData } from '../../core/errors/index.js';
import type { Logger } from '../../core/logger/core.js';
import { isRemotePath, removeLeadingForwardSlash } from '../../core/path.js';
import { isServerLikeOutput } from '../../prerender/utils.js';
import { isServerLikeOutput } from '../../core/util.js';
import type { MapValue } from '../../type-utils.js';
import { getConfiguredImageService } from '../internal.js';
import type { LocalImageService } from '../services/service.js';
Expand Down
2 changes: 1 addition & 1 deletion packages/astro/src/assets/vite-plugin-assets.ts
Expand Up @@ -12,7 +12,7 @@ import {
removeBase,
removeQueryString,
} from '../core/path.js';
import { isServerLikeOutput } from '../prerender/utils.js';
import { isServerLikeOutput } from '../core/util.js';
import { VALID_INPUT_FORMATS, VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js';
import { emitESMImage } from './utils/emitAsset.js';
import { getAssetsPrefix } from './utils/getAssetsPrefix.js';
Expand Down
3 changes: 2 additions & 1 deletion packages/astro/src/content/runtime.ts
Expand Up @@ -44,15 +44,16 @@ export function createCollectionToGlobResultMap({
return collectionToGlobResultMap;
}

const cacheEntriesByCollection = new Map<string, any[]>();
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was creating global state making it impossible to run the same fixture test multiple times with different config.

export function createGetCollection({
contentCollectionToEntryMap,
dataCollectionToEntryMap,
getRenderEntryImport,
cacheEntriesByCollection,
}: {
contentCollectionToEntryMap: CollectionToEntryMap;
dataCollectionToEntryMap: CollectionToEntryMap;
getRenderEntryImport: GetEntryImport;
cacheEntriesByCollection: Map<string, any[]>;
}) {
return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
let type: 'content' | 'data';
Expand Down
2 changes: 1 addition & 1 deletion packages/astro/src/content/vite-plugin-content-imports.ts
Expand Up @@ -15,7 +15,7 @@ import type {
import { getProxyCode } from '../assets/utils/proxy.js';
import { AstroError } from '../core/errors/errors.js';
import { AstroErrorData } from '../core/errors/index.js';
import { isServerLikeOutput } from '../prerender/utils.js';
import { isServerLikeOutput } from '../core/util.js';
import { CONTENT_FLAG, DATA_FLAG } from './consts.js';
import {
type ContentConfig,
Expand Down
Expand Up @@ -8,8 +8,7 @@ import type { AstroSettings } from '../@types/astro.js';
import { encodeName } from '../core/build/util.js';
import { AstroError, AstroErrorData } from '../core/errors/index.js';
import { appendForwardSlash, removeFileExtension } from '../core/path.js';
import { rootRelativePath } from '../core/util.js';
import { isServerLikeOutput } from '../prerender/utils.js';
import { rootRelativePath, isServerLikeOutput } from '../core/util.js';
import type { AstroPluginMetadata } from '../vite-plugin-astro/index.js';
import {
CONTENT_FLAG,
Expand Down
4 changes: 2 additions & 2 deletions packages/astro/src/core/build/generate.ts
Expand Up @@ -31,7 +31,7 @@ import {
} from '../../core/path.js';
import { toRoutingStrategy } from '../../i18n/utils.js';
import { runHookBuildGenerated } from '../../integrations/hooks.js';
import { getOutputDirectory, isServerLikeOutput } from '../../prerender/utils.js';
import { getOutputDirectory } from '../../prerender/utils.js';
import type { SSRManifestI18n } from '../app/types.js';
import { NoPrerenderedRoutesWithDomains } from '../errors/errors-data.js';
import { AstroError, AstroErrorData } from '../errors/index.js';
Expand All @@ -45,7 +45,7 @@ import { RenderContext } from '../render-context.js';
import { callGetStaticPaths } from '../render/route-cache.js';
import { createRequest } from '../request.js';
import { matchRoute } from '../routing/match.js';
import { getOutputFilename } from '../util.js';
import { getOutputFilename, isServerLikeOutput } from '../util.js';
import { getOutDirWithinCwd, getOutFile, getOutFolder } from './common.js';
import {
cssOrder,
Expand Down
3 changes: 1 addition & 2 deletions packages/astro/src/core/build/index.ts
Expand Up @@ -19,7 +19,6 @@ import {
runHookConfigDone,
runHookConfigSetup,
} from '../../integrations/hooks.js';
import { isServerLikeOutput } from '../../prerender/utils.js';
import { resolveConfig } from '../config/config.js';
import { createNodeLogger } from '../config/logging.js';
import { createSettings } from '../config/settings.js';
Expand All @@ -28,7 +27,7 @@ import type { Logger } from '../logger/core.js';
import { levels, timerMessage } from '../logger/core.js';
import { apply as applyPolyfill } from '../polyfill.js';
import { createRouteManifest } from '../routing/index.js';
import { ensureProcessNodeEnv } from '../util.js';
import { ensureProcessNodeEnv, isServerLikeOutput } from '../util.js';
import { collectPagesData } from './page-data.js';
import { staticBuild, viteBuild } from './static-build.js';
import type { StaticBuildOptions } from './types.js';
Expand Down
3 changes: 3 additions & 0 deletions packages/astro/src/core/build/internal.ts
Expand Up @@ -25,6 +25,8 @@ export interface BuildInternals {
hoistedScriptIdToHoistedMap: Map<string, Set<string>>;
// A mapping of hoisted script ids back to the pages which reference it
hoistedScriptIdToPagesMap: Map<string, Set<string>>;
// A mapping of hoisted script ids back to the content which reference it
hoistedScriptIdToContentMap: Map<string, Set<string>>;

/**
* Used by the `directRenderScript` option. If script is inlined, its id and
Expand Down Expand Up @@ -123,6 +125,7 @@ export function createBuildInternals(): BuildInternals {
cssModuleToChunkIdMap: new Map(),
hoistedScriptIdToHoistedMap,
hoistedScriptIdToPagesMap,
hoistedScriptIdToContentMap: new Map(),
inlinedScripts: new Map(),
entrySpecifierToBundleMap: new Map<string, string>(),
pageToBundleMap: new Map<string, string>(),
Expand Down
3 changes: 2 additions & 1 deletion packages/astro/src/core/build/pipeline.ts
@@ -1,5 +1,5 @@
import type { RouteData, SSRLoadedRenderer, SSRResult } from '../../@types/astro.js';
import { getOutputDirectory, isServerLikeOutput } from '../../prerender/utils.js';
import { getOutputDirectory } from '../../prerender/utils.js';
import { BEFORE_HYDRATION_SCRIPT_ID, PAGE_SCRIPT_ID } from '../../vite-plugin-scripts/index.js';
import type { SSRManifest } from '../app/types.js';
import { routeIsFallback, routeIsRedirect } from '../redirects/helpers.js';
Expand All @@ -21,6 +21,7 @@ import { getVirtualModulePageNameFromPath } from './plugins/util.js';
import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js';
import type { PageBuildData, StaticBuildOptions } from './types.js';
import { i18nHasFallback } from './util.js';
import { isServerLikeOutput } from '../util.js';

/**
* The build pipeline is responsible to gather the files emitted by the SSR build and generate the pages by executing these files.
Expand Down
91 changes: 65 additions & 26 deletions packages/astro/src/core/build/plugins/plugin-analyzer.ts
Expand Up @@ -6,6 +6,7 @@ import type { AstroBuildPlugin } from '../plugin.js';

import { PROPAGATED_ASSET_FLAG } from '../../../content/consts.js';
import { prependForwardSlash } from '../../../core/path.js';
import { isContentCollectionsCacheEnabled } from '../../../core/util.js';
import {
getParentModuleInfos,
getTopLevelPageModuleInfos,
Expand All @@ -31,6 +32,7 @@ export function vitePluginAnalyzer(
const pageScripts = new Map<
string,
{
type: 'page' | 'content';
hoistedSet: Set<string>;
propagatedMapByImporter: Map<string, Set<string>>;
}
Expand All @@ -51,27 +53,46 @@ export function vitePluginAnalyzer(
if (hoistedScripts.size) {
for (const parentInfo of getParentModuleInfos(from, this, isPropagatedAsset)) {
if (isPropagatedAsset(parentInfo.id)) {
for (const nestedParentInfo of getParentModuleInfos(from, this)) {
if (moduleIsTopLevelPage(nestedParentInfo)) {
for (const hid of hoistedScripts) {
if (!pageScripts.has(nestedParentInfo.id)) {
pageScripts.set(nestedParentInfo.id, {
hoistedSet: new Set(),
propagatedMapByImporter: new Map(),
});
}
const entry = pageScripts.get(nestedParentInfo.id)!;
if (!entry.propagatedMapByImporter.has(parentInfo.id)) {
entry.propagatedMapByImporter.set(parentInfo.id, new Set());
if(isContentCollectionsCacheEnabled(options.settings.config)) {
if (!pageScripts.has(parentInfo.id)) {
pageScripts.set(parentInfo.id, {
type: 'content',
hoistedSet: new Set(),
propagatedMapByImporter: new Map(),
});
}
const propagaters = pageScripts.get(parentInfo.id)!.propagatedMapByImporter;
for(const hid of hoistedScripts) {
if(!propagaters.has(parentInfo.id)) {
propagaters.set(parentInfo.id, new Set());
}
propagaters.get(parentInfo.id)!.add(hid);
}
} else {
for (const nestedParentInfo of getParentModuleInfos(from, this)) {
if (moduleIsTopLevelPage(nestedParentInfo)) {
for (const hid of hoistedScripts) {
if (!pageScripts.has(nestedParentInfo.id)) {
pageScripts.set(nestedParentInfo.id, {
type: 'page',
hoistedSet: new Set(),
propagatedMapByImporter: new Map(),
});
}
const entry = pageScripts.get(nestedParentInfo.id)!;
if (!entry.propagatedMapByImporter.has(parentInfo.id)) {
entry.propagatedMapByImporter.set(parentInfo.id, new Set());
}
entry.propagatedMapByImporter.get(parentInfo.id)!.add(hid);
}
entry.propagatedMapByImporter.get(parentInfo.id)!.add(hid);
}
}
}
} else if (moduleIsTopLevelPage(parentInfo)) {
for (const hid of hoistedScripts) {
if (!pageScripts.has(parentInfo.id)) {
pageScripts.set(parentInfo.id, {
type: 'page',
hoistedSet: new Set(),
propagatedMapByImporter: new Map(),
});
Expand All @@ -84,12 +105,21 @@ export function vitePluginAnalyzer(
},

finalize() {
for (const [pageId, { hoistedSet, propagatedMapByImporter }] of pageScripts) {
const pageData = getPageDataByViteID(internals, pageId);
if (!pageData) continue;
for (const [pageId, { hoistedSet, propagatedMapByImporter, type }] of pageScripts) {
let astroModuleId: string;
if(type === 'page') {
const pageData = getPageDataByViteID(internals, pageId);
if (!pageData) {
continue;
};
const { component } = pageData;
astroModuleId = prependForwardSlash(component);

const { component } = pageData;
const astroModuleId = prependForwardSlash(component);
// Keep track of the importers
pageData.propagatedScripts = propagatedMapByImporter;
} else {
astroModuleId = pageId;
}

const uniqueHoistedId = JSON.stringify(Array.from(hoistedSet).sort());
let moduleId: string;
Expand All @@ -104,8 +134,6 @@ export function vitePluginAnalyzer(
}
internals.discoveredScripts.add(moduleId);

pageData.propagatedScripts = propagatedMapByImporter;

// Add propagated scripts to client build,
// but DON'T add to pages -> hoisted script map.
for (const propagatedScripts of propagatedMapByImporter.values()) {
Expand All @@ -114,13 +142,24 @@ export function vitePluginAnalyzer(
}
}

// Make sure to track that this page uses this set of hoisted scripts
if (internals.hoistedScriptIdToPagesMap.has(moduleId)) {
const pages = internals.hoistedScriptIdToPagesMap.get(moduleId);
pages!.add(astroModuleId);
if(type === 'page') {
// Make sure to track that this page uses this set of hoisted scripts
if (internals.hoistedScriptIdToPagesMap.has(moduleId)) {
const pages = internals.hoistedScriptIdToPagesMap.get(moduleId);
pages!.add(astroModuleId);
} else {
internals.hoistedScriptIdToPagesMap.set(moduleId, new Set([astroModuleId]));
internals.hoistedScriptIdToHoistedMap.set(moduleId, hoistedSet);
}
} else {
internals.hoistedScriptIdToPagesMap.set(moduleId, new Set([astroModuleId]));
internals.hoistedScriptIdToHoistedMap.set(moduleId, hoistedSet);
// For content collections save to hoistedScriptIdToContentMap instead
if (internals.hoistedScriptIdToContentMap.has(moduleId)) {
const contentModules = internals.hoistedScriptIdToContentMap.get(moduleId);
contentModules!.add(astroModuleId);
} else {
internals.hoistedScriptIdToContentMap.set(moduleId, new Set([astroModuleId]));
internals.hoistedScriptIdToHoistedMap.set(moduleId, hoistedSet);
}
}
}
},
Expand Down