Skip to content

Commit

Permalink
Preserve content modules properly in cache (#10889)
Browse files Browse the repository at this point in the history 
* Wait until after build to preserve content modules

* Properly build hoisted scripts

* Add changeset

* Fix tests hitting each other

* Global state, the shame

* Move the file copying over to earlier in the process
  • Loading branch information
@matthewp
matthewp committed May 6, 2024
1 parent 082abb8 commit 4d905cc
Show file tree
Hide file tree
Showing 33 changed files with 385 additions and 345 deletions.
5 changes: 5 additions & 0 deletions .changeset/spicy-keys-own.md
View file
@@ -0,0 +1,5 @@
---
"astro": patch
---

Preserve content modules properly in cache
2 changes: 2 additions & 0 deletions packages/astro/content-module.template.mjs
View file
Expand Up @@ -48,10 +48,12 @@ const collectionToRenderEntryMap = createCollectionToGlobResultMap({
contentDir,
});

const cacheEntriesByCollection = new Map();
export const getCollection = createGetCollection({
contentCollectionToEntryMap,
dataCollectionToEntryMap,
getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap),
cacheEntriesByCollection,
});

export const getEntryBySlug = createGetEntryBySlug({
Expand Down
2 changes: 1 addition & 1 deletion packages/astro/src/assets/build/generate.ts
View file
Expand Up @@ -10,7 +10,7 @@ import { AstroError } from '../../core/errors/errors.js';
import { AstroErrorData } from '../../core/errors/index.js';
import type { Logger } from '../../core/logger/core.js';
import { isRemotePath, removeLeadingForwardSlash } from '../../core/path.js';
import { isServerLikeOutput } from '../../prerender/utils.js';
import { isServerLikeOutput } from '../../core/util.js';
import type { MapValue } from '../../type-utils.js';
import { getConfiguredImageService } from '../internal.js';
import type { LocalImageService } from '../services/service.js';
Expand Down
2 changes: 1 addition & 1 deletion packages/astro/src/assets/vite-plugin-assets.ts
View file
Expand Up @@ -12,7 +12,7 @@ import {
removeBase,
removeQueryString,
} from '../core/path.js';
import { isServerLikeOutput } from '../prerender/utils.js';
import { isServerLikeOutput } from '../core/util.js';
import { VALID_INPUT_FORMATS, VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js';
import { emitESMImage } from './utils/emitAsset.js';
import { getAssetsPrefix } from './utils/getAssetsPrefix.js';
Expand Down
3 changes: 2 additions & 1 deletion packages/astro/src/content/runtime.ts
View file
Expand Up @@ -44,15 +44,16 @@ export function createCollectionToGlobResultMap({
return collectionToGlobResultMap;
}

const cacheEntriesByCollection = new Map<string, any[]>();
export function createGetCollection({
contentCollectionToEntryMap,
dataCollectionToEntryMap,
getRenderEntryImport,
cacheEntriesByCollection,
}: {
contentCollectionToEntryMap: CollectionToEntryMap;
dataCollectionToEntryMap: CollectionToEntryMap;
getRenderEntryImport: GetEntryImport;
cacheEntriesByCollection: Map<string, any[]>;
}) {
return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
let type: 'content' | 'data';
Expand Down
2 changes: 1 addition & 1 deletion packages/astro/src/content/vite-plugin-content-imports.ts
View file
Expand Up @@ -15,7 +15,7 @@ import type {
import { getProxyCode } from '../assets/utils/proxy.js';
import { AstroError } from '../core/errors/errors.js';
import { AstroErrorData } from '../core/errors/index.js';
import { isServerLikeOutput } from '../prerender/utils.js';
import { isServerLikeOutput } from '../core/util.js';
import { CONTENT_FLAG, DATA_FLAG } from './consts.js';
import {
type ContentConfig,
Expand Down
3 changes: 1 addition & 2 deletions packages/astro/src/content/vite-plugin-content-virtual-mod.ts
View file
Expand Up @@ -8,8 +8,7 @@ import type { AstroSettings } from '../@types/astro.js';
import { encodeName } from '../core/build/util.js';
import { AstroError, AstroErrorData } from '../core/errors/index.js';
import { appendForwardSlash, removeFileExtension } from '../core/path.js';
import { rootRelativePath } from '../core/util.js';
import { isServerLikeOutput } from '../prerender/utils.js';
import { rootRelativePath, isServerLikeOutput } from '../core/util.js';
import type { AstroPluginMetadata } from '../vite-plugin-astro/index.js';
import {
CONTENT_FLAG,
Expand Down
4 changes: 2 additions & 2 deletions packages/astro/src/core/build/generate.ts
View file
Expand Up @@ -31,7 +31,7 @@ import {
} from '../../core/path.js';
import { toRoutingStrategy } from '../../i18n/utils.js';
import { runHookBuildGenerated } from '../../integrations/hooks.js';
import { getOutputDirectory, isServerLikeOutput } from '../../prerender/utils.js';
import { getOutputDirectory } from '../../prerender/utils.js';
import type { SSRManifestI18n } from '../app/types.js';
import { NoPrerenderedRoutesWithDomains } from '../errors/errors-data.js';
import { AstroError, AstroErrorData } from '../errors/index.js';
Expand All @@ -45,7 +45,7 @@ import { RenderContext } from '../render-context.js';
import { callGetStaticPaths } from '../render/route-cache.js';
import { createRequest } from '../request.js';
import { matchRoute } from '../routing/match.js';
import { getOutputFilename } from '../util.js';
import { getOutputFilename, isServerLikeOutput } from '../util.js';
import { getOutDirWithinCwd, getOutFile, getOutFolder } from './common.js';
import {
cssOrder,
Expand Down
3 changes: 1 addition & 2 deletions packages/astro/src/core/build/index.ts
View file
Expand Up @@ -19,7 +19,6 @@ import {
runHookConfigDone,
runHookConfigSetup,
} from '../../integrations/hooks.js';
import { isServerLikeOutput } from '../../prerender/utils.js';
import { resolveConfig } from '../config/config.js';
import { createNodeLogger } from '../config/logging.js';
import { createSettings } from '../config/settings.js';
Expand All @@ -28,7 +27,7 @@ import type { Logger } from '../logger/core.js';
import { levels, timerMessage } from '../logger/core.js';
import { apply as applyPolyfill } from '../polyfill.js';
import { createRouteManifest } from '../routing/index.js';
import { ensureProcessNodeEnv } from '../util.js';
import { ensureProcessNodeEnv, isServerLikeOutput } from '../util.js';
import { collectPagesData } from './page-data.js';
import { staticBuild, viteBuild } from './static-build.js';
import type { StaticBuildOptions } from './types.js';
Expand Down
3 changes: 3 additions & 0 deletions packages/astro/src/core/build/internal.ts
View file
Expand Up @@ -25,6 +25,8 @@ export interface BuildInternals {
hoistedScriptIdToHoistedMap: Map<string, Set<string>>;
// A mapping of hoisted script ids back to the pages which reference it
hoistedScriptIdToPagesMap: Map<string, Set<string>>;
// A mapping of hoisted script ids back to the content which reference it
hoistedScriptIdToContentMap: Map<string, Set<string>>;

/**
* Used by the `directRenderScript` option. If script is inlined, its id and
Expand Down Expand Up @@ -123,6 +125,7 @@ export function createBuildInternals(): BuildInternals {
cssModuleToChunkIdMap: new Map(),
hoistedScriptIdToHoistedMap,
hoistedScriptIdToPagesMap,
hoistedScriptIdToContentMap: new Map(),
inlinedScripts: new Map(),
entrySpecifierToBundleMap: new Map<string, string>(),
pageToBundleMap: new Map<string, string>(),
Expand Down
3 changes: 2 additions & 1 deletion packages/astro/src/core/build/pipeline.ts
View file
@@ -1,5 +1,5 @@
import type { RouteData, SSRLoadedRenderer, SSRResult } from '../../@types/astro.js';
import { getOutputDirectory, isServerLikeOutput } from '../../prerender/utils.js';
import { getOutputDirectory } from '../../prerender/utils.js';
import { BEFORE_HYDRATION_SCRIPT_ID, PAGE_SCRIPT_ID } from '../../vite-plugin-scripts/index.js';
import type { SSRManifest } from '../app/types.js';
import { routeIsFallback, routeIsRedirect } from '../redirects/helpers.js';
Expand All @@ -21,6 +21,7 @@ import { getVirtualModulePageNameFromPath } from './plugins/util.js';
import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js';
import type { PageBuildData, StaticBuildOptions } from './types.js';
import { i18nHasFallback } from './util.js';
import { isServerLikeOutput } from '../util.js';

/**
* The build pipeline is responsible to gather the files emitted by the SSR build and generate the pages by executing these files.
Expand Down
91 changes: 65 additions & 26 deletions packages/astro/src/core/build/plugins/plugin-analyzer.ts
View file
Expand Up @@ -6,6 +6,7 @@ import type { AstroBuildPlugin } from '../plugin.js';

import { PROPAGATED_ASSET_FLAG } from '../../../content/consts.js';
import { prependForwardSlash } from '../../../core/path.js';
import { isContentCollectionsCacheEnabled } from '../../../core/util.js';
import {
getParentModuleInfos,
getTopLevelPageModuleInfos,
Expand All @@ -31,6 +32,7 @@ export function vitePluginAnalyzer(
const pageScripts = new Map<
string,
{
type: 'page' | 'content';
hoistedSet: Set<string>;
propagatedMapByImporter: Map<string, Set<string>>;
}
Expand All @@ -51,27 +53,46 @@ export function vitePluginAnalyzer(
if (hoistedScripts.size) {
for (const parentInfo of getParentModuleInfos(from, this, isPropagatedAsset)) {
if (isPropagatedAsset(parentInfo.id)) {
for (const nestedParentInfo of getParentModuleInfos(from, this)) {
if (moduleIsTopLevelPage(nestedParentInfo)) {
for (const hid of hoistedScripts) {
if (!pageScripts.has(nestedParentInfo.id)) {
pageScripts.set(nestedParentInfo.id, {
hoistedSet: new Set(),
propagatedMapByImporter: new Map(),
});
}
const entry = pageScripts.get(nestedParentInfo.id)!;
if (!entry.propagatedMapByImporter.has(parentInfo.id)) {
entry.propagatedMapByImporter.set(parentInfo.id, new Set());
if(isContentCollectionsCacheEnabled(options.settings.config)) {
if (!pageScripts.has(parentInfo.id)) {
pageScripts.set(parentInfo.id, {
type: 'content',
hoistedSet: new Set(),
propagatedMapByImporter: new Map(),
});
}
const propagaters = pageScripts.get(parentInfo.id)!.propagatedMapByImporter;
for(const hid of hoistedScripts) {
if(!propagaters.has(parentInfo.id)) {
propagaters.set(parentInfo.id, new Set());
}
propagaters.get(parentInfo.id)!.add(hid);
}
} else {
for (const nestedParentInfo of getParentModuleInfos(from, this)) {
if (moduleIsTopLevelPage(nestedParentInfo)) {
for (const hid of hoistedScripts) {
if (!pageScripts.has(nestedParentInfo.id)) {
pageScripts.set(nestedParentInfo.id, {
type: 'page',
hoistedSet: new Set(),
propagatedMapByImporter: new Map(),
});
}
const entry = pageScripts.get(nestedParentInfo.id)!;
if (!entry.propagatedMapByImporter.has(parentInfo.id)) {
entry.propagatedMapByImporter.set(parentInfo.id, new Set());
}
entry.propagatedMapByImporter.get(parentInfo.id)!.add(hid);
}
entry.propagatedMapByImporter.get(parentInfo.id)!.add(hid);
}
}
}
} else if (moduleIsTopLevelPage(parentInfo)) {
for (const hid of hoistedScripts) {
if (!pageScripts.has(parentInfo.id)) {
pageScripts.set(parentInfo.id, {
type: 'page',
hoistedSet: new Set(),
propagatedMapByImporter: new Map(),
});
Expand All @@ -84,12 +105,21 @@ export function vitePluginAnalyzer(
},

finalize() {
for (const [pageId, { hoistedSet, propagatedMapByImporter }] of pageScripts) {
const pageData = getPageDataByViteID(internals, pageId);
if (!pageData) continue;
for (const [pageId, { hoistedSet, propagatedMapByImporter, type }] of pageScripts) {
let astroModuleId: string;
if(type === 'page') {
const pageData = getPageDataByViteID(internals, pageId);
if (!pageData) {
continue;
};
const { component } = pageData;
astroModuleId = prependForwardSlash(component);

const { component } = pageData;
const astroModuleId = prependForwardSlash(component);
// Keep track of the importers
pageData.propagatedScripts = propagatedMapByImporter;
} else {
astroModuleId = pageId;
}

const uniqueHoistedId = JSON.stringify(Array.from(hoistedSet).sort());
let moduleId: string;
Expand All @@ -104,8 +134,6 @@ export function vitePluginAnalyzer(
}
internals.discoveredScripts.add(moduleId);

pageData.propagatedScripts = propagatedMapByImporter;

// Add propagated scripts to client build,
// but DON'T add to pages -> hoisted script map.
for (const propagatedScripts of propagatedMapByImporter.values()) {
Expand All @@ -114,13 +142,24 @@ export function vitePluginAnalyzer(
}
}

// Make sure to track that this page uses this set of hoisted scripts
if (internals.hoistedScriptIdToPagesMap.has(moduleId)) {
const pages = internals.hoistedScriptIdToPagesMap.get(moduleId);
pages!.add(astroModuleId);
if(type === 'page') {
// Make sure to track that this page uses this set of hoisted scripts
if (internals.hoistedScriptIdToPagesMap.has(moduleId)) {
const pages = internals.hoistedScriptIdToPagesMap.get(moduleId);
pages!.add(astroModuleId);
} else {
internals.hoistedScriptIdToPagesMap.set(moduleId, new Set([astroModuleId]));
internals.hoistedScriptIdToHoistedMap.set(moduleId, hoistedSet);
}
} else {
internals.hoistedScriptIdToPagesMap.set(moduleId, new Set([astroModuleId]));
internals.hoistedScriptIdToHoistedMap.set(moduleId, hoistedSet);
// For content collections save to hoistedScriptIdToContentMap instead
if (internals.hoistedScriptIdToContentMap.has(moduleId)) {
const contentModules = internals.hoistedScriptIdToContentMap.get(moduleId);
contentModules!.add(astroModuleId);
} else {
internals.hoistedScriptIdToContentMap.set(moduleId, new Set([astroModuleId]));
internals.hoistedScriptIdToHoistedMap.set(moduleId, hoistedSet);
}
}
}
},
Expand Down

0 comments on commit 4d905cc

Please sign in to comment.