Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix static generation and crawler requests #41735

Merged
merged 5 commits into from Oct 24, 2022
Merged

Fix static generation and crawler requests #41735

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0c1bff2
wip
shuding Oct 24, 2022
090e7d2
update
shuding Oct 24, 2022
4851087
fix ssg
shuding Oct 24, 2022
160f4b7
fix fetch
ijjk Oct 24, 2022
6067bd6
fix
shuding Oct 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
52 changes: 46 additions & 6 deletions packages/next/server/app-render.tsx
View file
Open in desktop
Expand Up @@ -145,6 +145,7 @@ export type RenderOptsPartial = {
serverComponents?: boolean
assetPrefix?: string
fontLoaderManifest?: FontLoaderManifest
isBot?: boolean
}

export type RenderOpts = LoadComponentsReturnType & RenderOptsPartial
Expand Down Expand Up @@ -175,9 +176,12 @@ function createErrorHandler(
* Used for debugging
*/
_source: string,
capturedErrors: Error[]
capturedErrors: Error[],
allCapturedErrors?: Error[]
) {
return (err: any): string => {
if (allCapturedErrors) allCapturedErrors.push(err)

if (
err.digest === DYNAMIC_ERROR_CODE ||
err.digest === NOT_FOUND_ERROR_CODE ||
Expand Down Expand Up @@ -703,10 +707,12 @@ export async function renderToHTMLOrFlight(
* These rules help ensure that other existing features like request caching,
* coalescing, and ISR continue working as intended.
*/
const isStaticGeneration = renderOpts.supportsDynamicHTML !== true
const isStaticGeneration =
renderOpts.supportsDynamicHTML !== true && !renderOpts.isBot
const isFlight = req.headers.__rsc__ !== undefined

const capturedErrors: Error[] = []
const allCapturedErrors: Error[] = []

const serverComponentsErrorHandler = createErrorHandler(
'serverComponentsRenderer',
Expand All @@ -718,7 +724,8 @@ export async function renderToHTMLOrFlight(
)
const htmlRendererErrorHandler = createErrorHandler(
'htmlRenderer',
capturedErrors
capturedErrors,
allCapturedErrors
)

const {
Expand All @@ -729,9 +736,11 @@ export async function renderToHTMLOrFlight(
ComponentMod,
dev,
fontLoaderManifest,
supportsDynamicHTML,
} = renderOpts

patchFetch(ComponentMod)
const generateStaticHTML = supportsDynamicHTML !== true

const staticGenerationAsyncStorage = ComponentMod.staticGenerationAsyncStorage
const requestAsyncStorage = ComponentMod.requestAsyncStorage
Expand Down Expand Up @@ -1508,20 +1517,51 @@ export async function renderToHTMLOrFlight(
},
})

return await continueFromInitialStream(renderStream, {
const result = await continueFromInitialStream(renderStream, {
dataStream: serverComponentsInlinedTransformStream?.readable,
generateStaticHTML: isStaticGeneration,
generateStaticHTML: isStaticGeneration || generateStaticHTML,
getServerInsertedHTML,
serverInsertedHTMLToHead: true,
...validateRootLayout,
})

if (generateStaticHTML) {
let html = await streamToString(result)

if (
allCapturedErrors.some(
(e: any) => e.digest === NOT_FOUND_ERROR_CODE
)
) {
// If a not found error is thrown, we return 404 and make sure to
// inject the noindex tag.
res.statusCode = 404
html = html.replace(
'<head>',
'<head><meta name="robots" content="noindex"/>'
)
}

return html
}

return result
} catch (err: any) {
const shouldNotIndex = err.digest === NOT_FOUND_ERROR_CODE
if (err.digest === NOT_FOUND_ERROR_CODE) {
res.statusCode = 404
}

// TODO-APP: show error overlay in development. `element` should probably be wrapped in AppRouter for this case.
const renderStream = await renderToInitialStream({
ReactDOMServer,
element: (
<html id="__next_error__">
<head></head>
<head>
{shouldNotIndex ? (
<meta name="robots" content="noindex" />
) : null}
</head>
<body></body>
</html>
),
Expand Down
3 changes: 3 additions & 0 deletions packages/next/server/base-server.ts
View file
Open in desktop
Expand Up @@ -205,6 +205,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
serverComponents?: boolean
crossOrigin?: string
supportsDynamicHTML?: boolean
isBot?: boolean
serverComponentManifest?: any
serverCSSManifest?: any
fontLoaderManifest?: FontLoaderManifest
Expand Down Expand Up @@ -868,6 +869,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
renderOpts: {
...this.renderOpts,
supportsDynamicHTML: !isBotRequest,
isBot: !!isBotRequest,
},
} as const
const payload = await fn(ctx)
Expand Down Expand Up @@ -1160,6 +1162,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
// cache if there are no dynamic data requirements
opts.supportsDynamicHTML =
!isSSG && !isBotRequest && !query.amp && isSupportedDocument
opts.isBot = isBotRequest
}

const defaultLocale = isSSG
Expand Down
1 change: 1 addition & 0 deletions packages/next/server/render.tsx
View file
Open in desktop
Expand Up @@ -253,6 +253,7 @@ export type RenderOptsPartial = {
domainLocales?: DomainLocale[]
disableOptimizedLoading?: boolean
supportsDynamicHTML?: boolean
isBot?: boolean
runtime?: ServerRuntime
serverComponents?: boolean
customServer?: boolean
Expand Down
18 changes: 18 additions & 0 deletions test/e2e/app-dir/index.test.ts
View file
Open in desktop
Expand Up @@ -1899,6 +1899,24 @@ describe('app dir', () => {
})
})

describe('bots', () => {
it('should block rendering for bots and return 404 status', async () => {
const res = await fetchViaHTTP(
next.url,
'/not-found/servercomponent',
'',
{
headers: {
'User-Agent': 'Googlebot',
},
}
)

expect(res.status).toBe(404)
expect(await res.text()).toInclude('"noindex"')
})
})

describe('redirect', () => {
describe('components', () => {
it('should redirect in a server component', async () => {
Expand Down