diff --git a/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts b/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts index d8b8447ce3bb2a4..c34360b06560042 100644 --- a/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts +++ b/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts @@ -69,7 +69,9 @@ export class TraceEntryPointsPlugin implements webpack.Plugin { const traceOutputName = `${isWebpack5 ? '../' : ''}${ entrypoint.name }.js.nft.json` - const traceOutputPath = nodePath.join(outputPath, traceOutputName) + const traceOutputPath = nodePath.dirname( + nodePath.join(outputPath, traceOutputName) + ) assets[traceOutputName] = new sources.RawSource( JSON.stringify({ diff --git a/packages/next/package.json b/packages/next/package.json index 4559184fe8b25d3..680b93d5c9fee44 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -50,7 +50,8 @@ "scripts": { "dev": "taskr", "release": "taskr release", - "prepublish": "npm run release && yarn types", + "prepublish": "npm run release && yarn types && yarn trace-server", + "trace-server": "node ../../scripts/trace-next-server.js", "types": "tsc --declaration --emitDeclarationOnly --declarationDir dist", "typescript": "tsc --noEmit --declaration", "ncc-compiled": "ncc cache clean && taskr ncc", diff --git a/packages/next/server/next-server.ts b/packages/next/server/next-server.ts index 63e280f2375834d..fc32aadf7bbb8f1 100644 --- a/packages/next/server/next-server.ts +++ b/packages/next/server/next-server.ts @@ -87,7 +87,6 @@ import { FontManifest } from './font-utils' import { denormalizePagePath } from './denormalize-page-path' import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path' import * as Log from '../build/output/log' -import { imageOptimizer } from './image-optimizer' import { detectDomainLocale } from '../shared/lib/i18n/detect-domain-locale' import escapePathDelimiters from '../shared/lib/router/utils/escape-path-delimiters' import { getUtils } from '../build/webpack/loaders/next-serverless-loader/utils' @@ -700,8 +699,18 @@ export default class Server { match: route('/_next/image'), type: 'route', name: '_next/image catchall', - fn: (req, res, _params, parsedUrl) => - imageOptimizer( + fn: (req, res, _params, parsedUrl) => { + if (this.minimalMode) { + res.statusCode = 400 + res.end('Bad Request') + return { + finished: true, + } + } + const { imageOptimizer } = + require('./image-optimizer') as typeof import('./image-optimizer') + + return imageOptimizer( server, req, res, @@ -709,7 +718,8 @@ export default class Server { server.nextConfig, server.distDir, this.renderOpts.dev - ), + ) + }, }, { match: route('/_next/:path*'), diff --git a/packages/next/taskfile.js b/packages/next/taskfile.js index 5d1d84572ad0a1f..4598476a909df7b 100644 --- a/packages/next/taskfile.js +++ b/packages/next/taskfile.js @@ -1,9 +1,7 @@ -const fs = require('fs') // eslint-disable-next-line import/no-extraneous-dependencies const notifier = require('node-notifier') // eslint-disable-next-line import/no-extraneous-dependencies -const { nodeFileTrace } = require('@vercel/nft') -const { join, relative, basename, resolve } = require('path') +const { relative, basename, resolve } = require('path') const { Module } = require('module') // Note: @@ -764,45 +762,6 @@ export async function precompile(task, opts) { ) } -// eslint-disable-next-line camelcase -export async function trace_next_server(task) { - const { TRACE_OUTPUT_VERSION } = require('next/dist/shared/lib/constants') - const root = join(__dirname, '../../') - - const result = await nodeFileTrace( - [require.resolve('next/dist/server/next-server')], - { - base: root, - processCwd: __dirname, - ignore: [ - 'packages/next/dist/compiled/webpack/(bundle4|bundle5).js', - 'node_modules/react/**/*.development.js', - 'node_modules/react-dom/**/*.development.js', - 'node_modules/use-subscription/**/*.development.js', - 'node_modules/next/dist/next-server/server/lib/squoosh/**/*.wasm', - 'packages/next/dist/pages/**/*', - ], - } - ) - - const tracedDeps = [] - - for (const file of result.fileList) { - if (result.reasons[file].type === 'initial') { - continue - } - tracedDeps.push(join(root, file)) - } - - fs.writeFileSync( - join(__dirname, 'dist/server/next-server.nft.json'), - JSON.stringify({ - version: TRACE_OUTPUT_VERSION, - files: tracedDeps, - }) - ) -} - // eslint-disable-next-line camelcase export async function copy_ncced(task) { // we don't ncc every time we build since these won't change @@ -1000,7 +959,7 @@ export async function telemetry(task, opts) { } export async function build(task, opts) { - await task.serial(['precompile', 'compile', 'trace_next_server'], opts) + await task.serial(['precompile', 'compile'], opts) } export default async function (task) { diff --git a/scripts/trace-next-server.js b/scripts/trace-next-server.js new file mode 100644 index 000000000000000..016f62f90496870 --- /dev/null +++ b/scripts/trace-next-server.js @@ -0,0 +1,132 @@ +const os = require('os') +const path = require('path') +const execa = require('execa') +const fs = require('fs-extra') +const prettyBytes = require('pretty-bytes') +const gzipSize = require('next/dist/compiled/gzip-size') +const { nodeFileTrace } = require('next/dist/compiled/@vercel/nft') + +const MAX_COMPRESSED_SIZE = 250 * 1000 +const MAX_UNCOMPRESSED_SIZE = 2.5 * 1000 * 1000 + +// install next outside the monorepo for clean `node_modules` +// to trace against which helps ensure minimal trace is +// produced. +// react and react-dom need to be traced specific to installed +// version so isn't pre-traced +async function main() { + const tmpdir = os.tmpdir() + await execa('yarn', ['pack'], { + cwd: path.join(__dirname, '../packages/next'), + stdio: ['ignore', 'inherit', 'inherit'], + }) + const packagePath = path.join( + __dirname, + `../packages/next/next-v${ + require('../packages/next/package.json').version + }.tgz` + ) + const workDir = path.join(tmpdir, `trace-next-${Date.now()}`) + console.log('using workdir', workDir) + await fs.ensureDir(workDir) + + await fs.writeFile( + path.join(workDir, 'package.json'), + JSON.stringify( + { + dependencies: { + next: packagePath, + }, + private: true, + }, + null, + 2 + ) + ) + await execa('yarn', ['install'], { + cwd: workDir, + stdio: ['ignore', 'inherit', 'inherit'], + env: { + ...process.env, + YARN_CACHE_FOLDER: path.join(workDir, '.yarn-cache'), + }, + }) + + const nextServerPath = path.join( + workDir, + 'node_modules/next/dist/server/next-server.js' + ) + + const traceLabel = `traced ${nextServerPath}` + console.time(traceLabel) + + const result = await nodeFileTrace([nextServerPath], { + base: workDir, + processCwd: workDir, + ignore: [ + 'node_modules/next/dist/pages/**/*', + 'node_modules/next/dist/server/image-optimizer.js', + 'node_modules/next/dist/compiled/@ampproject/toolbox-optimizer/**/*', + 'node_modules/next/dist/server/lib/squoosh/**/*.wasm', + 'node_modules/next/dist/compiled/webpack/(bundle4|bundle5).js', + 'node_modules/react/**/*.development.js', + 'node_modules/react-dom/**/*.development.js', + 'node_modules/use-subscription/**/*.development.js', + 'node_modules/sharp/**/*', + ], + }) + + const tracedDeps = new Set() + let totalCompressedSize = 0 + let totalUncompressedSize = 0 + + for (const file of result.fileList) { + if (result.reasons[file].type === 'initial') { + continue + } + tracedDeps.add(file) + const stat = await fs.stat(path.join(workDir, file)) + + if (stat.isFile()) { + const compressedSize = await gzipSize(path.join(workDir, file)) + totalUncompressedSize += stat.size || 0 + totalCompressedSize += compressedSize + } else { + console.log('not a file', file, stat.isDirectory()) + } + } + + console.log({ + numberFiles: tracedDeps.size, + totalGzipSize: prettyBytes(totalCompressedSize), + totalUncompressedSize: prettyBytes(totalUncompressedSize), + }) + + await fs.writeFile( + path.join( + __dirname, + '../packages/next/dist/server/next-server.js.nft.json' + ), + JSON.stringify({ + files: Array.from(tracedDeps), + version: 1, + }) + ) + await fs.unlink(packagePath) + await fs.remove(workDir) + + console.timeEnd(traceLabel) + + if ( + totalCompressedSize > MAX_COMPRESSED_SIZE || + totalUncompressedSize > MAX_UNCOMPRESSED_SIZE + ) { + throw new Error( + `Max traced size of next-server exceeded limits of ${MAX_COMPRESSED_SIZE} compressed or ${MAX_UNCOMPRESSED_SIZE} uncompressed` + ) + } +} + +main() + .then(() => console.log('done')) + .catch(console.error) diff --git a/test/integration/required-server-files/lib/config.js b/test/integration/required-server-files/app/lib/config.js similarity index 100% rename from test/integration/required-server-files/lib/config.js rename to test/integration/required-server-files/app/lib/config.js diff --git a/test/integration/required-server-files/next.config.js b/test/integration/required-server-files/app/next.config.js similarity index 70% rename from test/integration/required-server-files/next.config.js rename to test/integration/required-server-files/app/next.config.js index 3636a73ef1f1b1c..88e42a12b22f4bf 100644 --- a/test/integration/required-server-files/next.config.js +++ b/test/integration/required-server-files/app/next.config.js @@ -1,6 +1,12 @@ module.exports = { // ensure incorrect target is overridden by env target: 'serverless', + experimental: { + nftTracing: true, + }, + eslint: { + ignoreDuringBuilds: true, + }, rewrites() { return [ { diff --git a/test/integration/required-server-files/pages/[slug].js b/test/integration/required-server-files/app/pages/[slug].js similarity index 100% rename from test/integration/required-server-files/pages/[slug].js rename to test/integration/required-server-files/app/pages/[slug].js diff --git a/test/integration/required-server-files/pages/_app.js b/test/integration/required-server-files/app/pages/_app.js similarity index 100% rename from test/integration/required-server-files/pages/_app.js rename to test/integration/required-server-files/app/pages/_app.js diff --git a/test/integration/required-server-files/pages/api/error.js b/test/integration/required-server-files/app/pages/api/error.js similarity index 100% rename from test/integration/required-server-files/pages/api/error.js rename to test/integration/required-server-files/app/pages/api/error.js diff --git a/test/integration/required-server-files/pages/api/optional/[[...rest]].js b/test/integration/required-server-files/app/pages/api/optional/[[...rest]].js similarity index 100% rename from test/integration/required-server-files/pages/api/optional/[[...rest]].js rename to test/integration/required-server-files/app/pages/api/optional/[[...rest]].js diff --git a/test/integration/required-server-files/pages/catch-all/[[...rest]].js b/test/integration/required-server-files/app/pages/catch-all/[[...rest]].js similarity index 100% rename from test/integration/required-server-files/pages/catch-all/[[...rest]].js rename to test/integration/required-server-files/app/pages/catch-all/[[...rest]].js diff --git a/test/integration/required-server-files/pages/dynamic/[slug].js b/test/integration/required-server-files/app/pages/dynamic/[slug].js similarity index 100% rename from test/integration/required-server-files/pages/dynamic/[slug].js rename to test/integration/required-server-files/app/pages/dynamic/[slug].js diff --git a/test/integration/required-server-files/pages/errors/gip.js b/test/integration/required-server-files/app/pages/errors/gip.js similarity index 100% rename from test/integration/required-server-files/pages/errors/gip.js rename to test/integration/required-server-files/app/pages/errors/gip.js diff --git a/test/integration/required-server-files/pages/errors/gsp/[post].js b/test/integration/required-server-files/app/pages/errors/gsp/[post].js similarity index 100% rename from test/integration/required-server-files/pages/errors/gsp/[post].js rename to test/integration/required-server-files/app/pages/errors/gsp/[post].js diff --git a/test/integration/required-server-files/pages/errors/gssp.js b/test/integration/required-server-files/app/pages/errors/gssp.js similarity index 100% rename from test/integration/required-server-files/pages/errors/gssp.js rename to test/integration/required-server-files/app/pages/errors/gssp.js diff --git a/test/integration/required-server-files/pages/fallback/[slug].js b/test/integration/required-server-files/app/pages/fallback/[slug].js similarity index 100% rename from test/integration/required-server-files/pages/fallback/[slug].js rename to test/integration/required-server-files/app/pages/fallback/[slug].js diff --git a/test/integration/required-server-files/pages/index.js b/test/integration/required-server-files/app/pages/index.js similarity index 100% rename from test/integration/required-server-files/pages/index.js rename to test/integration/required-server-files/app/pages/index.js diff --git a/test/integration/required-server-files/pages/optional-ssg/[[...rest]].js b/test/integration/required-server-files/app/pages/optional-ssg/[[...rest]].js similarity index 100% rename from test/integration/required-server-files/pages/optional-ssg/[[...rest]].js rename to test/integration/required-server-files/app/pages/optional-ssg/[[...rest]].js diff --git a/test/integration/required-server-files/pages/optional-ssp/[[...rest]].js b/test/integration/required-server-files/app/pages/optional-ssp/[[...rest]].js similarity index 100% rename from test/integration/required-server-files/pages/optional-ssp/[[...rest]].js rename to test/integration/required-server-files/app/pages/optional-ssp/[[...rest]].js diff --git a/test/integration/required-server-files/test/index.test.js b/test/integration/required-server-files/test/index.test.js index 4275d1a9e81e6fb..6c90be9c5253a57 100644 --- a/test/integration/required-server-files/test/index.test.js +++ b/test/integration/required-server-files/test/index.test.js @@ -1,20 +1,25 @@ /* eslint-env jest */ -import http from 'http' +import os from 'os' +import glob from 'glob' import fs from 'fs-extra' -import { join } from 'path' +import execa from 'execa' import cheerio from 'cheerio' -import { nextServer } from 'next-test-utils' +import { join, dirname, relative } from 'path' +import { version } from 'next/package' +import { recursiveReadDir } from 'next/dist/lib/recursive-readdir' import { fetchViaHTTP, findPort, - nextBuild, + initNextServerScript, + killApp, renderViaHTTP, } from 'next-test-utils' jest.setTimeout(1000 * 60 * 2) -const appDir = join(__dirname, '..') +const appDir = join(__dirname, '../app') +const workDir = join(os.tmpdir(), `required-server-files-${Date.now()}`) let server let nextApp let appPort @@ -24,59 +29,175 @@ let errors = [] describe('Required Server Files', () => { beforeAll(async () => { - await fs.remove(join(appDir, '.next')) - await nextBuild(appDir, undefined, { + const nextServerTrace = await fs.readJSON( + require.resolve('next/dist/server/next-server') + '.nft.json' + ) + const packageDir = dirname(require.resolve('next/package.json')) + await execa('yarn', ['pack'], { + cwd: packageDir, + }) + const packagePath = join(packageDir, `next-v${version}.tgz`) + + await fs.ensureDir(workDir) + await fs.writeFile( + join(workDir, 'package.json'), + JSON.stringify({ + dependencies: { + next: packagePath, + react: 'latest', + 'react-dom': 'latest', + }, + }) + ) + await fs.copy(appDir, workDir) + + await execa('yarn', ['install'], { + cwd: workDir, + stdio: ['ignore', 'inherit', 'inherit'], env: { + ...process.env, + YARN_CACHE_FOLDER: join(workDir, '.yarn-cache'), + }, + }) + + await execa('yarn', ['next', 'build'], { + cwd: workDir, + stdio: ['ignore', 'inherit', 'inherit'], + env: { + ...process.env, + NODE_ENV: 'production', NOW_BUILDER: '1', }, }) - buildId = await fs.readFile(join(appDir, '.next/BUILD_ID'), 'utf8') + buildId = await fs.readFile(join(workDir, '.next/BUILD_ID'), 'utf8') requiredFilesManifest = await fs.readJSON( - join(appDir, '.next/required-server-files.json') + join(workDir, '.next/required-server-files.json') + ) + + // react and react-dom need to be traced specific to version + // so isn't pre-traced + await fs.ensureDir(`${workDir}-react`) + await fs.writeFile( + join(`${workDir}-react/package.json`), + JSON.stringify({ + dependencies: { + react: 'latest', + 'react-dom': 'latest', + }, + }) ) + await execa('yarn', ['install'], { + cwd: `${workDir}-react`, + stdio: ['ignore', 'inherit', 'inherit'], + env: { + ...process.env, + YARN_CACHE_FOLDER: join(workDir, '.yarn'), + }, + }) + await fs.remove(packagePath) - let files = await fs.readdir(join(appDir, '.next')) + const files = await recursiveReadDir(workDir, /.*/) + + const pageTraceFiles = await glob.sync('**/*.nft.json', { + cwd: join(workDir, '.next/server/pages'), + }) + const combinedTraces = new Set() + + for (const file of pageTraceFiles) { + const filePath = join(workDir, '.next/server/pages', file) + const trace = await fs.readJSON(filePath) + + trace.files.forEach((f) => + combinedTraces.add(relative(workDir, join(dirname(filePath), f))) + ) + } for (const file of files) { + const cleanFile = join('./', file) if ( - file === 'server' || - file === 'required-server-files.json' || - requiredFilesManifest.files.includes(join('.next', file)) + !nextServerTrace.files.includes(cleanFile) && + file !== '/node_modules/next/dist/server/next-server.js' && + !combinedTraces.has(cleanFile) && + !requiredFilesManifest.files.includes(cleanFile) && + !cleanFile.startsWith('.next/server') && + cleanFile !== '.next/required-server-files.json' ) { - continue + await fs.remove(join(workDir, file)) } - console.log('removing', join('.next', file)) - await fs.remove(join(appDir, '.next', file)) } - await fs.rename(join(appDir, 'pages'), join(appDir, 'pages-bak')) - nextApp = nextServer({ - conf: {}, - dir: appDir, - quiet: false, - minimalMode: true, - }) - appPort = await findPort() + for (const file of await fs.readdir(`${workDir}-react/node_modules`)) { + await fs.copy( + join(`${workDir}-react/node_modules`, file), + join(workDir, 'node_modules', file) + ) + } + await fs.remove(`${workDir}-react`) + + async function startServer() { + const http = require('http') + const NextServer = require('next/dist/server/next-server').default + + const appPort = process.env.PORT + nextApp = new NextServer({ + conf: global.nextConfig, + dir: process.env.APP_DIR, + quiet: false, + minimalMode: true, + }) + + server = http.createServer(async (req, res) => { + try { + await nextApp.getRequestHandler()(req, res) + } catch (err) { + console.error('top-level', err) + res.statusCode = 500 + res.end('error') + } + }) + await new Promise((res, rej) => { + server.listen(appPort, (err) => (err ? rej(err) : res())) + }) + console.log(`Listening at ::${appPort}`) + } + + const serverPath = join(workDir, 'server.js') + + await fs.writeFile( + serverPath, + 'global.nextConfig = ' + + JSON.stringify(requiredFilesManifest.config) + + ';\n' + + startServer.toString() + + ';\n' + + `startServer().catch(console.error)` + ) - server = http.createServer(async (req, res) => { - try { - await nextApp.getRequestHandler()(req, res) - } catch (err) { - console.error('top-level', err) - errors.push(err) - res.statusCode = 500 - res.end('error') + appPort = await findPort() + server = await initNextServerScript( + serverPath, + /Listening at/, + { + ...process.env, + NODE_ENV: 'production', + PORT: appPort, + APP_DIR: workDir, + }, + undefined, + { + cwd: workDir, + onStderr(msg) { + if (msg.includes('top-level')) { + errors.push(msg) + } + }, } - }) - await new Promise((res, rej) => { - server.listen(appPort, (err) => (err ? rej(err) : res())) - }) - console.log(`Listening at ::${appPort}`) + ) }) afterAll(async () => { - if (server) server.close() - await fs.rename(join(appDir, 'pages-bak'), join(appDir, 'pages')) + if (server) killApp(server) + await fs.remove(workDir) }) it('should output required-server-files manifest correctly', async () => { @@ -90,11 +211,9 @@ describe('Required Server Files', () => { expect(typeof requiredFilesManifest.appDir).toBe('string') for (const file of requiredFilesManifest.files) { - console.log('checking', file) - expect(await fs.exists(join(appDir, file))).toBe(true) + expect(await fs.exists(join(workDir, file))).toBe(true) } - - expect(await fs.exists(join(appDir, '.next/server'))).toBe(true) + expect(await fs.exists(join(workDir, '.next/server'))).toBe(true) }) it('should render SSR page correctly', async () => { @@ -449,7 +568,7 @@ describe('Required Server Files', () => { expect(res.status).toBe(500) expect(await res.text()).toBe('error') expect(errors.length).toBe(1) - expect(errors[0].message).toContain('gip hit an oops') + expect(errors[0]).toContain('gip hit an oops') }) it('should bubble error correctly for gssp page', async () => { @@ -458,7 +577,7 @@ describe('Required Server Files', () => { expect(res.status).toBe(500) expect(await res.text()).toBe('error') expect(errors.length).toBe(1) - expect(errors[0].message).toContain('gssp hit an oops') + expect(errors[0]).toContain('gssp hit an oops') }) it('should bubble error correctly for gsp page', async () => { @@ -467,7 +586,7 @@ describe('Required Server Files', () => { expect(res.status).toBe(500) expect(await res.text()).toBe('error') expect(errors.length).toBe(1) - expect(errors[0].message).toContain('gsp hit an oops') + expect(errors[0]).toContain('gsp hit an oops') }) it('should bubble error correctly for API page', async () => { @@ -476,7 +595,7 @@ describe('Required Server Files', () => { expect(res.status).toBe(500) expect(await res.text()).toBe('error') expect(errors.length).toBe(1) - expect(errors[0].message).toContain('some error from /api/error') + expect(errors[0]).toContain('some error from /api/error') }) it('should normalize optional values correctly for SSP page', async () => {