diff --git a/packages/next/build/babel/loader/index.ts b/packages/next/build/babel/loader/index.ts index 6fea5706fa355ae..5c3615b61241422 100644 --- a/packages/next/build/babel/loader/index.ts +++ b/packages/next/build/babel/loader/index.ts @@ -40,7 +40,7 @@ const nextBabelLoaderOuter = function nextBabelLoaderOuter( ) { const callback = this.async() - const loaderSpan = trace('next-babel-turbo-loader', this.currentTraceSpan?.id) + const loaderSpan = this.currentTraceSpan.traceChild('next-babel-turbo-loader') loaderSpan .traceAsyncFn(() => nextBabelLoader.call(this, loaderSpan, inputSource, inputSourceMap) diff --git a/packages/next/build/babel/loader/types.d.ts b/packages/next/build/babel/loader/types.d.ts index 200a4780f3bca27..895ea04077ff56b 100644 --- a/packages/next/build/babel/loader/types.d.ts +++ b/packages/next/build/babel/loader/types.d.ts @@ -2,7 +2,7 @@ import { loader } from 'next/dist/compiled/webpack/webpack' import { Span } from '../../../telemetry/trace' export interface NextJsLoaderContext extends loader.LoaderContext { - currentTraceSpan?: Span + currentTraceSpan: Span } export interface NextBabelLoaderOptions { diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts index cf26b1dfcd9fe02..679f34323ce84df 100644 --- a/packages/next/build/index.ts +++ b/packages/next/build/index.ts @@ -130,11 +130,13 @@ export default async function build( const config: NextConfigComplete = await nextBuildSpan .traceChild('load-next-config') .traceAsyncFn(() => loadConfig(PHASE_PRODUCTION_BUILD, dir, conf)) + const distDir = path.join(dir, config.distDir) + setGlobal('distDir', distDir) + const { target } = config const buildId: string = await nextBuildSpan .traceChild('generate-buildid') .traceAsyncFn(() => generateBuildId(config.generateBuildId, nanoid)) - const distDir = path.join(dir, config.distDir) const customRoutes: CustomRoutes = await nextBuildSpan .traceChild('load-custom-routes') diff --git a/packages/next/build/webpack-config.ts b/packages/next/build/webpack-config.ts index 950ce3f59faef24..766d0776c33f95f 100644 --- a/packages/next/build/webpack-config.ts +++ b/packages/next/build/webpack-config.ts @@ -240,7 +240,7 @@ export default async function getBaseWebpackConfig( entrypoints: WebpackEntrypoints rewrites: CustomRoutes['rewrites'] isDevFallback?: boolean - runWebpackSpan?: Span + runWebpackSpan: Span } ): Promise { const hasRewrites = diff --git a/packages/next/build/webpack/loaders/next-serverless-loader/index.ts b/packages/next/build/webpack/loaders/next-serverless-loader/index.ts index d64ece16177d60e..054def2f9feb97f 100644 --- a/packages/next/build/webpack/loaders/next-serverless-loader/index.ts +++ b/packages/next/build/webpack/loaders/next-serverless-loader/index.ts @@ -11,7 +11,6 @@ import { ROUTES_MANIFEST, REACT_LOADABLE_MANIFEST, } from '../../../../shared/lib/constants' -import { trace } from '../../../../telemetry/trace' export type ServerlessLoaderQuery = { page: string @@ -34,63 +33,61 @@ export type ServerlessLoaderQuery = { } const nextServerlessLoader: webpack.loader.Loader = function () { - const loaderSpan = trace('next-serverless-loader') - return loaderSpan.traceFn(() => { - const { - distDir, - absolutePagePath, - page, - buildId, - canonicalBase, - assetPrefix, - absoluteAppPath, - absoluteDocumentPath, - absoluteErrorPath, - absolute404Path, - generateEtags, - poweredByHeader, - basePath, - runtimeConfig, - previewProps, - loadedEnvFiles, - i18n, - }: ServerlessLoaderQuery = - typeof this.query === 'string' ? parse(this.query.substr(1)) : this.query - - const buildManifest = join(distDir, BUILD_MANIFEST).replace(/\\/g, '/') - const reactLoadableManifest = join( - distDir, - REACT_LOADABLE_MANIFEST - ).replace(/\\/g, '/') - const routesManifest = join(distDir, ROUTES_MANIFEST).replace(/\\/g, '/') - - const escapedBuildId = escapeRegexp(buildId) - const pageIsDynamicRoute = isDynamicRoute(page) - - const encodedPreviewProps = devalue( - JSON.parse(previewProps) as __ApiPreviewProps - ) - - const envLoading = ` + const { + distDir, + absolutePagePath, + page, + buildId, + canonicalBase, + assetPrefix, + absoluteAppPath, + absoluteDocumentPath, + absoluteErrorPath, + absolute404Path, + generateEtags, + poweredByHeader, + basePath, + runtimeConfig, + previewProps, + loadedEnvFiles, + i18n, + }: ServerlessLoaderQuery = + typeof this.query === 'string' ? parse(this.query.substr(1)) : this.query + + const buildManifest = join(distDir, BUILD_MANIFEST).replace(/\\/g, '/') + const reactLoadableManifest = join(distDir, REACT_LOADABLE_MANIFEST).replace( + /\\/g, + '/' + ) + const routesManifest = join(distDir, ROUTES_MANIFEST).replace(/\\/g, '/') + + const escapedBuildId = escapeRegexp(buildId) + const pageIsDynamicRoute = isDynamicRoute(page) + + const encodedPreviewProps = devalue( + JSON.parse(previewProps) as __ApiPreviewProps + ) + + const envLoading = ` const { processEnv } = require('@next/env') processEnv(${Buffer.from(loadedEnvFiles, 'base64').toString()}) ` - const runtimeConfigImports = runtimeConfig - ? ` + const runtimeConfigImports = runtimeConfig + ? ` const { setConfig } = require('next/config') ` - : '' + : '' - const runtimeConfigSetter = runtimeConfig - ? ` + const runtimeConfigSetter = runtimeConfig + ? ` const runtimeConfig = ${runtimeConfig} setConfig(runtimeConfig) ` - : 'const runtimeConfig = {}' + : 'const runtimeConfig = {}' - if (page.match(API_ROUTE)) { - return ` + if (page.match(API_ROUTE)) { + return ` ${envLoading} ${runtimeConfigImports} ${ @@ -125,8 +122,8 @@ const nextServerlessLoader: webpack.loader.Loader = function () { }) export default apiHandler ` - } else { - return ` + } else { + return ` import 'next/dist/server/node-polyfill-fetch' import routesManifest from '${routesManifest}' import buildManifest from '${buildManifest}' @@ -206,8 +203,7 @@ const nextServerlessLoader: webpack.loader.Loader = function () { }) export { renderReqToHTML, render } ` - } - }) + } } export default nextServerlessLoader diff --git a/packages/next/build/webpack/loaders/next-swc-loader.js b/packages/next/build/webpack/loaders/next-swc-loader.js index cd0cb1162062be0..51958b03e55585f 100644 --- a/packages/next/build/webpack/loaders/next-swc-loader.js +++ b/packages/next/build/webpack/loaders/next-swc-loader.js @@ -123,7 +123,7 @@ async function loaderTransform(parentTrace, source, inputSourceMap) { } export default function swcLoader(inputSource, inputSourceMap) { - const loaderSpan = trace('next-swc-loader', this.currentTraceSpan?.id) + const loaderSpan = this.currentTraceSpan.traceChild('next-swc-loader') const callback = this.async() loaderSpan .traceAsyncFn(() => diff --git a/packages/next/build/webpack/plugins/build-stats-plugin.ts b/packages/next/build/webpack/plugins/build-stats-plugin.ts index 3f88e6433595f6d..3b39bab799b1d60 100644 --- a/packages/next/build/webpack/plugins/build-stats-plugin.ts +++ b/packages/next/build/webpack/plugins/build-stats-plugin.ts @@ -118,7 +118,7 @@ export default class BuildStatsPlugin { async (stats, callback) => { const compilerSpan = spans.get(compiler) try { - const writeStatsSpan = trace('NextJsBuildStats', compilerSpan?.id) + const writeStatsSpan = compilerSpan!.traceChild('NextJsBuildStats') await writeStatsSpan.traceAsyncFn(() => { return new Promise((resolve, reject) => { const statsJson = reduceSize( diff --git a/packages/next/build/webpack/plugins/css-minimizer-plugin.ts b/packages/next/build/webpack/plugins/css-minimizer-plugin.ts index 77d40c8c3430cb2..df676ad1cd95cb3 100644 --- a/packages/next/build/webpack/plugins/css-minimizer-plugin.ts +++ b/packages/next/build/webpack/plugins/css-minimizer-plugin.ts @@ -71,9 +71,8 @@ export class CssMinimizerPlugin { }, async (assets: any) => { const compilerSpan = spans.get(compiler) - const cssMinimizerSpan = trace( - 'css-minimizer-plugin', - compilerSpan?.id + const cssMinimizerSpan = compilerSpan!.traceChild( + 'css-minimizer-plugin' ) cssMinimizerSpan.setAttribute('webpackVersion', 5) @@ -83,7 +82,7 @@ export class CssMinimizerPlugin { files .filter((file) => CSS_REGEX.test(file)) .map(async (file) => { - const assetSpan = trace('minify-css', cssMinimizerSpan.id) + const assetSpan = cssMinimizerSpan.traceChild('minify-css') assetSpan.setAttribute('file', file) return assetSpan.traceAsyncFn(async () => { diff --git a/packages/next/build/webpack/plugins/profiling-plugin.ts b/packages/next/build/webpack/plugins/profiling-plugin.ts index e8c131da0329daa..6379277f85232f3 100644 --- a/packages/next/build/webpack/plugins/profiling-plugin.ts +++ b/packages/next/build/webpack/plugins/profiling-plugin.ts @@ -15,9 +15,9 @@ function getNormalModuleLoaderHook(compilation: any) { export class ProfilingPlugin { compiler: any - runWebpackSpan: Span | undefined + runWebpackSpan: Span - constructor({ runWebpackSpan }: { runWebpackSpan: Span | undefined }) { + constructor({ runWebpackSpan }: { runWebpackSpan: Span }) { this.runWebpackSpan = runWebpackSpan } apply(compiler: any) { @@ -35,14 +35,17 @@ export class ProfilingPlugin { attrs, onSetSpan, }: { - parentSpan?: () => Span | undefined + parentSpan?: () => Span attrs?: any onSetSpan?: (span: Span) => void } = {} ) { let span: Span | undefined startHook.tap(pluginName, () => { - span = trace(spanName, parentSpan?.()?.id, attrs ? attrs() : attrs) + span = parentSpan + ? parentSpan().traceChild(spanName, attrs ? attrs() : attrs) + : trace(spanName, undefined, attrs ? attrs() : attrs) + onSetSpan?.(span) }) stopHook.tap(pluginName, () => { @@ -103,10 +106,16 @@ export class ProfilingPlugin { const issuerModule = compilation?.moduleGraph?.getIssuer(module) - const span = trace( - `build-module${moduleType ? `-${moduleType}` : ''}`, - issuerModule ? spans.get(issuerModule)?.id : compilerSpan.id - ) + let span: Span + + const spanName = `build-module${moduleType ? `-${moduleType}` : ''}` + const issuerSpan: Span | undefined = + issuerModule && spans.get(issuerModule) + if (issuerSpan) { + span = issuerSpan.traceChild(spanName) + } else { + span = compilerSpan.traceChild(spanName) + } span.setAttribute('name', module.userRequest) spans.set(module, span) }) diff --git a/packages/next/server/dev/hot-reloader.ts b/packages/next/server/dev/hot-reloader.ts index 81fcdefd70eec38..277e6b958d9e5bc 100644 --- a/packages/next/server/dev/hot-reloader.ts +++ b/packages/next/server/dev/hot-reloader.ts @@ -27,6 +27,7 @@ import { difference } from '../../build/utils' import { NextConfigComplete } from '../config-shared' import { CustomRoutes } from '../../lib/load-custom-routes' import { DecodeError } from '../../shared/lib/utils' +import { Span, trace } from '../../telemetry/trace' export async function renderScriptError( res: ServerResponse, @@ -143,6 +144,7 @@ export default class HotReloader { private watcher: any private rewrites: CustomRoutes['rewrites'] private fallbackWatcher: any + private hotReloaderSpan: Span public isWebpack5: any constructor( @@ -174,6 +176,7 @@ export default class HotReloader { this.previewProps = previewProps this.rewrites = rewrites this.isWebpack5 = isWebpack5 + this.hotReloaderSpan = trace('hot-reloader') } public async run( @@ -283,6 +286,7 @@ export default class HotReloader { pagesDir: this.pagesDir, rewrites: this.rewrites, entrypoints: entrypoints.client, + runWebpackSpan: this.hotReloaderSpan, }), getBaseWebpackConfig(this.dir, { dev: true, @@ -292,6 +296,7 @@ export default class HotReloader { pagesDir: this.pagesDir, rewrites: this.rewrites, entrypoints: entrypoints.server, + runWebpackSpan: this.hotReloaderSpan, }), ]) } @@ -300,6 +305,7 @@ export default class HotReloader { if (this.fallbackWatcher) return const fallbackConfig = await getBaseWebpackConfig(this.dir, { + runWebpackSpan: this.hotReloaderSpan, dev: true, isServer: false, config: this.config, diff --git a/packages/next/telemetry/trace/report/index.ts b/packages/next/telemetry/trace/report/index.ts index 2f77f93b7fcdd9e..d817802d45356fc 100644 --- a/packages/next/telemetry/trace/report/index.ts +++ b/packages/next/telemetry/trace/report/index.ts @@ -3,6 +3,7 @@ import reportToConsole from './to-console' import reportToZipkin from './to-zipkin' import reportToJaeger from './to-jaeger' import reportToTelemetry from './to-telemetry' +import reportToJson from './to-json' type Reporter = { flushAll: () => Promise | void @@ -16,6 +17,31 @@ type Reporter = { ) => void } +class MultiReporter implements Reporter { + private reporters: Reporter[] = [] + + constructor(reporters: Reporter[]) { + this.reporters = reporters + } + + async flushAll() { + await Promise.all(this.reporters.map((reporter) => reporter.flushAll())) + } + + report( + spanName: string, + duration: number, + timestamp: number, + id: SpanId, + parentId?: SpanId, + attrs?: Object + ) { + this.reporters.forEach((reporter) => + reporter.report(spanName, duration, timestamp, id, parentId, attrs) + ) + } +} + const target = process.env.TRACE_TARGET && process.env.TRACE_TARGET in TARGET ? TARGET[process.env.TRACE_TARGET as TARGET] @@ -27,14 +53,17 @@ if (process.env.TRACE_TARGET && !target) { ) } -export let reporter: Reporter +let traceTargetReporter: Reporter if (target === TARGET.CONSOLE) { - reporter = reportToConsole + traceTargetReporter = reportToConsole } else if (target === TARGET.ZIPKIN) { - reporter = reportToZipkin + traceTargetReporter = reportToZipkin } else if (target === TARGET.JAEGER) { - reporter = reportToJaeger + traceTargetReporter = reportToJaeger } else { - reporter = reportToTelemetry + traceTargetReporter = reportToTelemetry } + +// JSON is always reported to allow for diagnostics +export const reporter = new MultiReporter([reportToJson, traceTargetReporter]) diff --git a/packages/next/telemetry/trace/report/to-jaeger.ts b/packages/next/telemetry/trace/report/to-jaeger.ts index 809c80b13cd1423..9ce257e616fb9a9 100644 --- a/packages/next/telemetry/trace/report/to-jaeger.ts +++ b/packages/next/telemetry/trace/report/to-jaeger.ts @@ -5,7 +5,7 @@ import * as Log from '../../../build/output/log' // Jaeger uses Zipkin's reporting import { batcher } from './to-zipkin' -let traceId = process.env.TRACE_ID +let traceId: string let batch: ReturnType | undefined const localEndpoint = { @@ -33,7 +33,7 @@ const reportToLocalHost = ( attrs?: Object ) => { if (!traceId) { - traceId = process.env.TRACE_ID = randomBytes(8).toString('hex') + traceId = process.env.TRACE_ID || randomBytes(8).toString('hex') logWebUrl() } diff --git a/packages/next/telemetry/trace/report/to-json.ts b/packages/next/telemetry/trace/report/to-json.ts new file mode 100644 index 000000000000000..473576681192cbd --- /dev/null +++ b/packages/next/telemetry/trace/report/to-json.ts @@ -0,0 +1,71 @@ +import { randomBytes } from 'crypto' +import { batcher } from './to-zipkin' +import { traceGlobals } from '../shared' +import fs from 'fs' +import path from 'path' + +let writeStream: fs.WriteStream +let traceId: string +let batch: ReturnType | undefined + +const reportToLocalHost = ( + name: string, + duration: number, + timestamp: number, + id: string, + parentId?: string, + attrs?: Object +) => { + const distDir = traceGlobals.get('distDir') + if (!distDir) { + return + } + + if (!traceId) { + traceId = process.env.TRACE_ID || randomBytes(8).toString('hex') + } + + if (!batch) { + batch = batcher(async (events) => { + if (!writeStream) { + const tracesDir = path.join(distDir, 'traces') + await fs.promises.mkdir(tracesDir, { recursive: true }) + const file = path.join(distDir, 'trace') + writeStream = fs.createWriteStream(file, { + flags: 'a', + encoding: 'utf8', + }) + } + const eventsJson = JSON.stringify(events) + try { + await new Promise((resolve, reject) => { + writeStream.write(eventsJson + '\n', 'utf8', (err) => { + err ? reject(err) : resolve() + }) + }) + } catch (err) { + console.log(err) + } + }) + } + + batch.report({ + traceId, + parentId, + name, + id, + timestamp, + duration, + tags: attrs, + }) +} + +export default { + flushAll: () => + batch + ? batch.flushAll().then(() => { + writeStream.end('', 'utf8') + }) + : undefined, + report: reportToLocalHost, +} diff --git a/packages/next/telemetry/trace/report/to-zipkin.ts b/packages/next/telemetry/trace/report/to-zipkin.ts index b34edec3eb6f3be..1fb0f5edec14a7f 100644 --- a/packages/next/telemetry/trace/report/to-zipkin.ts +++ b/packages/next/telemetry/trace/report/to-zipkin.ts @@ -3,7 +3,7 @@ import { randomBytes } from 'crypto' import fetch from 'node-fetch' import * as Log from '../../../build/output/log' -let traceId = process.env.TRACE_ID +let traceId: string let batch: ReturnType | undefined const localEndpoint = { @@ -21,7 +21,7 @@ type Event = { id: string timestamp: number duration: number - localEndpoint: typeof localEndpoint + localEndpoint?: typeof localEndpoint tags?: Object } @@ -42,8 +42,9 @@ export function batcher(reportEvents: (evts: Event[]) => Promise) { events.push(event) if (events.length > 100) { - const report = reportEvents(events.slice()) + const evts = events.slice() events.length = 0 + const report = reportEvents(evts) queue.add(report) report.then(() => queue.delete(report)) } @@ -60,7 +61,7 @@ const reportToLocalHost = ( attrs?: Object ) => { if (!traceId) { - traceId = process.env.TRACE_ID = randomBytes(8).toString('hex') + traceId = process.env.TRACE_ID || randomBytes(8).toString('hex') Log.info( `Zipkin trace will be available on ${zipkinUrl}/zipkin/traces/${traceId}` ) diff --git a/scripts/send-trace-to-jaeger.mjs b/scripts/send-trace-to-jaeger.mjs new file mode 100644 index 000000000000000..577b8c3e5c358a6 --- /dev/null +++ b/scripts/send-trace-to-jaeger.mjs @@ -0,0 +1,65 @@ +import fs from 'fs' +import eventStream from 'event-stream' +import retry from 'async-retry' +import fetch from 'node-fetch' + +const file = fs.createReadStream(process.argv[2]) + +const localEndpoint = { + serviceName: 'nextjs', + ipv4: '127.0.0.1', + port: 9411, +} + +// Jaeger supports Zipkin's reporting API +const zipkinUrl = `http://${localEndpoint.ipv4}:${localEndpoint.port}` +const jaegerWebUiUrl = `http://${localEndpoint.ipv4}:16686` +const zipkinAPI = `${zipkinUrl}/api/v2/spans` + +let loggedUrl = false + +function logWebUrl(traceId) { + console.log( + `Jaeger trace will be available on ${jaegerWebUiUrl}/trace/${traceId}` + ) +} +file.pipe(eventStream.split()).pipe( + eventStream.map((data, cb) => { + if (data === '') { + return cb(null, '') + } + + const eventsJson = JSON.parse(data).map((item) => { + item.localEndpoint = localEndpoint + return item + }) + if (!loggedUrl) { + logWebUrl(eventsJson[0].traceId) + loggedUrl = true + } + retry( + () => + // Send events to zipkin + fetch(zipkinAPI, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(eventsJson), + }), + { minTimeout: 500, retries: 3, factor: 1 } + ) + .then(async (res) => { + if (res.status !== 202) { + console.log({ + status: res.status, + body: await res.text(), + events: eventsJson, + }) + } + cb(null, '') + }) + .catch((err) => { + console.log(err) + cb(null, '') + }) + }) +)