From d6dc0c115c767846128b27d1207eed376565ad85 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sun, 17 Mar 2024 19:02:57 +0900 Subject: [PATCH 01/33] wip: prototype bench compare --- examples/basic/test/base.bench.ts | 22 ++++++ examples/basic/vite.config.ts | 115 +++++++++++++++++++++++++++++- 2 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 examples/basic/test/base.bench.ts diff --git a/examples/basic/test/base.bench.ts b/examples/basic/test/base.bench.ts new file mode 100644 index 000000000000..e059812d32fc --- /dev/null +++ b/examples/basic/test/base.bench.ts @@ -0,0 +1,22 @@ +import { bench, describe } from 'vitest' + +describe('sort', () => { + let x: number[] + let y: number[] + + bench('normal', () => { + x.sort((a, b) => a - b) + }, { + setup: () => { + x = Array.from({ length: 1000_000 }, (_, i) => i) + }, + }) + + bench('reverse', () => { + y.sort((a, b) => a - b) + }, { + setup: () => { + y = Array.from({ length: 1000_000 }, (_, i) => -i) + }, + }) +}) diff --git a/examples/basic/vite.config.ts b/examples/basic/vite.config.ts index 8bc6d4f1f0b8..07b05281376b 100644 --- a/examples/basic/vite.config.ts +++ b/examples/basic/vite.config.ts @@ -2,11 +2,122 @@ // Configure Vitest (https://vitest.dev/config/) +// pnpm -C examples/basic test bench -- --run + +// inspired by https://bheisler.github.io/criterion.rs/book/user_guide/command_line_options.html#baselines +// --save-baseline +// --baseline +// --load-baseline (no need?) + +import fs from 'node:fs' +import path from 'node:path' import { defineConfig } from 'vite' +import { BenchmarkReportsMap } from 'vitest/reporters' +import type { BenchmarkResult } from 'vitest' + +// node_modules/.vitest/bench/(save-baseline).json // default new.json +// node_modules/.vitest/bench/(baseline).json // default last new.json renambed to base.json +const benchDir = 'node_modules/.vitest/bench' + +class CompareReporter extends BenchmarkReportsMap.json { + // TODO: cli option? reporter option? env var? + constructor( + private options: { + baseline?: string // TODO: support multiple comparison? + saveBaseline?: string + } = {}, + ) { + super() + // for now, use env var for configuration + this.options.baseline = process.env.VITEST_BENCH_BASELINE + this.options.saveBaseline = process.env.VITEST_BENCH_SAVE_BASELINE + } + + async onFinished() { + if (fs.existsSync(benchDir)) + await fs.promises.mkdir(benchDir, { recursive: true }) + + let baseFile: string | undefined + let newFile = path.join(benchDir, 'new.json') + + if (this.options?.baseline) { + baseFile = path.join(benchDir, `${this.options.baseline}.json`) + if (!fs.existsSync(baseFile)) { + console.error('baseline not found:', baseFile) + return + } + } + else if (fs.existsSync(newFile)) { + // if no baseline provided, rename last new.json to base.json + baseFile = path.join(benchDir, 'base.json') + await fs.promises.copyFile(newFile, baseFile) + } + + if (this.options?.saveBaseline) + newFile = path.join(benchDir, `${this.options.saveBaseline}.json`) + + // reuse json reporter + this.ctx.config.benchmark!.outputFile = newFile + await super.onFinished() + + // output comparison with baseline + if (baseFile) + await compare(newFile, baseFile) + } +} + +async function compare(newFile: string, baseFile: string) { + console.log('') + console.log(' [BENCH] Comparison') + console.log(' current :', newFile) + console.log(' baseline :', baseFile) + console.log('') + const result = await parseResult(newFile) + const baseline = await parseResult(baseFile) + for (const [suite, tasks] of Object.entries(result)) { + console.log(suite) + for (const [name, curr] of Object.entries(tasks)) { + const base = baseline[suite]?.[name] + if (base) { + const diff = curr.hz / base.hz + console.log( + ' ', + name, + `${curr.hz.toPrecision(5)}hz`, + `[baseline: ${base.hz.toPrecision(5)}hz]`, + `[change: ${diff.toFixed(2)}x ${diff > 1 ? '⇑' : '⇓'}]`, + ) + } + else { + console.log( + ' ', + name, + `${curr.hz.toPrecision(5)}hz`, + '(no baseline)', + ) + } + } + } +} + +async function parseResult(file: string) { + // TODO: no bench filename? + interface RawResult { testResults: { [suite: string]: BenchmarkResult[] } } + const raw: RawResult = JSON.parse(await fs.promises.readFile(file, 'utf-8')) + const result: { [suite: string]: { [task: string]: BenchmarkResult } } = {} + for (const suite in raw.testResults) { + for (const task of raw.testResults[suite]) { + result[suite] ??= {} + result[suite][task.name] = task + } + } + return result +} export default defineConfig({ test: { - /* for example, use global to avoid globals imports (describe, test, expect): */ - // globals: true, + benchmark: { + reporters: ['default', new CompareReporter()], + }, }, }) From 6dca36ab877c6ac8653fef7d38386dc1156337b1 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Mon, 18 Mar 2024 08:39:28 +0900 Subject: [PATCH 02/33] wip --- examples/basic/vite.config.ts | 54 +++++++++++------------------------ 1 file changed, 16 insertions(+), 38 deletions(-) diff --git a/examples/basic/vite.config.ts b/examples/basic/vite.config.ts index 07b05281376b..a843a7b3e63e 100644 --- a/examples/basic/vite.config.ts +++ b/examples/basic/vite.config.ts @@ -15,54 +15,32 @@ import { defineConfig } from 'vite' import { BenchmarkReportsMap } from 'vitest/reporters' import type { BenchmarkResult } from 'vitest' -// node_modules/.vitest/bench/(save-baseline).json // default new.json -// node_modules/.vitest/bench/(baseline).json // default last new.json renambed to base.json -const benchDir = 'node_modules/.vitest/bench' - class CompareReporter extends BenchmarkReportsMap.json { - // TODO: cli option? reporter option? env var? - constructor( - private options: { - baseline?: string // TODO: support multiple comparison? - saveBaseline?: string - } = {}, - ) { - super() - // for now, use env var for configuration - this.options.baseline = process.env.VITEST_BENCH_BASELINE - this.options.saveBaseline = process.env.VITEST_BENCH_SAVE_BASELINE - } - async onFinished() { - if (fs.existsSync(benchDir)) - await fs.promises.mkdir(benchDir, { recursive: true }) + // TODO: use env var as flag for prototype - let baseFile: string | undefined - let newFile = path.join(benchDir, 'new.json') + // --compare + const baseFile = process.env.VITEST_BENCH_COMPARE; - if (this.options?.baseline) { - baseFile = path.join(benchDir, `${this.options.baseline}.json`) - if (!fs.existsSync(baseFile)) { - console.error('baseline not found:', baseFile) - return - } - } - else if (fs.existsSync(newFile)) { - // if no baseline provided, rename last new.json to base.json - baseFile = path.join(benchDir, 'base.json') - await fs.promises.copyFile(newFile, baseFile) - } + // writing is not necessary when --compare + // --benchmark.outputFile + const newFile = process.env.VITEST_BENCH_OUTPUT_FILE ?? "bench-default.json"; - if (this.options?.saveBaseline) - newFile = path.join(benchDir, `${this.options.saveBaseline}.json`) + if (fs.existsSync(path.dirname(newFile))) + await fs.promises.mkdir(path.dirname(newFile), { recursive: true }) - // reuse json reporter + // for now, reuse json reporter to save file this.ctx.config.benchmark!.outputFile = newFile await super.onFinished() - // output comparison with baseline - if (baseFile) + if (baseFile) { + if (!fs.existsSync(baseFile)) { + console.error('baseline not found:', baseFile) + return; + } + // output comparison with baseline await compare(newFile, baseFile) + } } } From 01ad3d761121862a9a621a3cd46c047246d04f82 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Mon, 18 Mar 2024 19:51:53 +0900 Subject: [PATCH 03/33] wip: benchmark comparison --- examples/basic/vite.config.ts | 148 +++++++++--------- .../src/node/reporters/benchmark/compare.ts | 101 ++++++++++++ 2 files changed, 179 insertions(+), 70 deletions(-) create mode 100644 packages/vitest/src/node/reporters/benchmark/compare.ts diff --git a/examples/basic/vite.config.ts b/examples/basic/vite.config.ts index a843a7b3e63e..d2f2cf6b8464 100644 --- a/examples/basic/vite.config.ts +++ b/examples/basic/vite.config.ts @@ -2,94 +2,102 @@ // Configure Vitest (https://vitest.dev/config/) -// pnpm -C examples/basic test bench -- --run - -// inspired by https://bheisler.github.io/criterion.rs/book/user_guide/command_line_options.html#baselines -// --save-baseline -// --baseline -// --load-baseline (no need?) - import fs from 'node:fs' import path from 'node:path' import { defineConfig } from 'vite' -import { BenchmarkReportsMap } from 'vitest/reporters' -import type { BenchmarkResult } from 'vitest' +import { Reporter } from 'vitest/reporters' +import type { BenchmarkResult, File, Task } from 'vitest' -class CompareReporter extends BenchmarkReportsMap.json { - async onFinished() { - // TODO: use env var as flag for prototype +/* - // --compare - const baseFile = process.env.VITEST_BENCH_COMPARE; +VITEST_BENCH_OUTPUT_FILE=main.json pnpm -C examples/basic test bench -- --run +VITEST_BENCH_COMPARE=main.json pnpm -C examples/basic test bench -- --run - // writing is not necessary when --compare - // --benchmark.outputFile - const newFile = process.env.VITEST_BENCH_OUTPUT_FILE ?? "bench-default.json"; +*/ - if (fs.existsSync(path.dirname(newFile))) - await fs.promises.mkdir(path.dirname(newFile), { recursive: true }) +type BenchEntries = { [id: string]: Omit }; - // for now, reuse json reporter to save file - this.ctx.config.benchmark!.outputFile = newFile - await super.onFinished() +function traverseTask( + task: Task, + callback: (task: Task, depth: number) => void, + depth = 0, +) { + if (task.type === "suite") { + callback(task, depth); + for (const t of task.tasks) { + traverseTask(t, callback, depth + 1); + } + } else { + callback(task, depth); + } +} + +export class CompareReporter implements Reporter { + async onFinished(files: File[] = []) { + // TODO: use env var as flag for prototype + // --benchmark.outputFile + const newFile = process.env.VITEST_BENCH_OUTPUT_FILE; + // --compare + const baseFile = process.env.VITEST_BENCH_COMPARE; + let baseEntries: BenchEntries | undefined; if (baseFile) { - if (!fs.existsSync(baseFile)) { + if (fs.existsSync(baseFile)) { + baseEntries = JSON.parse(await fs.promises.readFile(baseFile, "utf-8")); + } else { console.error('baseline not found:', baseFile) - return; } - // output comparison with baseline - await compare(newFile, baseFile) } - } -} -async function compare(newFile: string, baseFile: string) { - console.log('') - console.log(' [BENCH] Comparison') - console.log(' current :', newFile) - console.log(' baseline :', baseFile) - console.log('') - const result = await parseResult(newFile) - const baseline = await parseResult(baseFile) - for (const [suite, tasks] of Object.entries(result)) { - console.log(suite) - for (const [name, curr] of Object.entries(tasks)) { - const base = baseline[suite]?.[name] - if (base) { - const diff = curr.hz / base.hz - console.log( - ' ', - name, - `${curr.hz.toPrecision(5)}hz`, - `[baseline: ${base.hz.toPrecision(5)}hz]`, - `[change: ${diff.toFixed(2)}x ${diff > 1 ? '⇑' : '⇓'}]`, - ) - } - else { - console.log( - ' ', - name, - `${curr.hz.toPrecision(5)}hz`, - '(no baseline)', - ) - } + if (!newFile && !baseFile) { + console.log("use --benchmark.outputFile to save current run "); + return; + } + + if (baseFile) { + console.log('') + console.log('[BENCH] baseline:', path.resolve(baseFile)); + console.log('') + } + + const currentEntries: BenchEntries = {} + for (const file of files) { + traverseTask(file, (t, depth) => { + if (t.type === "suite") { + console.log(" ".repeat(depth * 2), t.name); + return; + } + if (t.result?.benchmark) { + const { samples, ...current } = t.result.benchmark + currentEntries[t.id] = current; + + const baseline = baseEntries?.[t.id]; + if (baseline) { + const diff = current.hz / baseline.hz; + console.log( + " ".repeat(depth * 2), + t.name, + `${current.hz.toPrecision(5)}hz`, + `[baseline: ${baseline.hz.toPrecision(5)}hz]`, + `[change: ${diff.toFixed(2)}x ${diff > 1 ? '⇑' : '⇓'}]`, + ) + } else { + console.log( + " ".repeat(depth * 2), + `${current.hz.toPrecision(5)}hz`, + `(no baseline)`, + ) + } + } + }) } - } -} -async function parseResult(file: string) { - // TODO: no bench filename? - interface RawResult { testResults: { [suite: string]: BenchmarkResult[] } } - const raw: RawResult = JSON.parse(await fs.promises.readFile(file, 'utf-8')) - const result: { [suite: string]: { [task: string]: BenchmarkResult } } = {} - for (const suite in raw.testResults) { - for (const task of raw.testResults[suite]) { - result[suite] ??= {} - result[suite][task.name] = task + if (newFile) { + if (!fs.existsSync(path.dirname(newFile))) + await fs.promises.mkdir(path.dirname(newFile), { recursive: true }) + await fs.promises.writeFile(newFile, JSON.stringify(currentEntries, null, 2)); } } - return result } export default defineConfig({ diff --git a/packages/vitest/src/node/reporters/benchmark/compare.ts b/packages/vitest/src/node/reporters/benchmark/compare.ts new file mode 100644 index 000000000000..083ab566b5da --- /dev/null +++ b/packages/vitest/src/node/reporters/benchmark/compare.ts @@ -0,0 +1,101 @@ +import fs from 'node:fs' +import path from 'node:path' +import type { BenchmarkResult, File, Reporter, Task, Vitest } from '../../../types' + +/* + +VITEST_BENCH_OUTPUT_FILE=main.json pnpm -C examples/basic test bench -- --run +VITEST_BENCH_COMPARE=main.json pnpm -C examples/basic test bench -- --run + +*/ + +type BenchEntries = { [id: string]: Omit }; + +function traverseTask( + task: Task, + callback: (task: Task, depth: number) => void, + depth = 0, +) { + if (task.type === "suite") { + callback(task, depth); + for (const t of task.tasks) { + traverseTask(t, callback, depth + 1); + } + } else { + callback(task, depth); + } +} + +export class CompareReporter implements Reporter { + ctx: Vitest = undefined! + + onInit(ctx: Vitest) { + this.ctx = ctx + } + + async onFinished(files: File[] = []) { + // TODO: use env var as flag for prototype + // --benchmark.outputFile + const newFile = process.env.VITEST_BENCH_OUTPUT_FILE; + // --compare + const baseFile = process.env.VITEST_BENCH_COMPARE; + + let baseEntries: BenchEntries | undefined; + if (baseFile) { + if (fs.existsSync(baseFile)) { + baseEntries = JSON.parse(await fs.promises.readFile(baseFile, "utf-8")); + } else { + console.error('baseline not found:', baseFile) + } + } + + if (!newFile && !baseFile) { + console.log("use --benchmark.outputFile to save current run "); + return; + } + + if (baseFile) { + console.log('') + console.log('[BENCH] baseline:', path.resolve(baseFile)); + console.log('') + } + + const currentEntries: BenchEntries = {} + for (const file of files) { + traverseTask(file, (t, depth) => { + if (t.type === "suite") { + this.ctx.logger.log(" ".repeat(depth * 2), t.name); + return; + } + if (t.result?.benchmark) { + const { samples: _samples, ...current } = t.result.benchmark + currentEntries[t.id] = current; + + const baseline = baseEntries?.[t.id]; + if (baseline) { + const diff = current.hz / baseline.hz; + this.ctx.logger.log( + " ".repeat(depth * 2), + t.name, + `${current.hz.toPrecision(5)}hz`, + `[baseline: ${baseline.hz.toPrecision(5)}hz]`, + `[change: ${diff.toFixed(2)}x ${diff > 1 ? '⇑' : '⇓'}]`, + ) + } else { + this.ctx.logger.log( + " ".repeat(depth * 2), + `${current.hz.toPrecision(5)}hz`, + `(no baseline)`, + ) + } + } + }) + } + + if (newFile) { + if (!fs.existsSync(path.dirname(newFile))) + await fs.promises.mkdir(path.dirname(newFile), { recursive: true }) + await fs.promises.writeFile(newFile, JSON.stringify(currentEntries, null, 2)); + } + } +} From 6d7b0a052a803b14b52ec17e505294b8698edeec Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 15:05:24 +0900 Subject: [PATCH 04/33] chore: cleanup example --- examples/basic/vite.config.ts | 101 +----------------- .../src/node/reporters/benchmark/compare.ts | 66 ++++++------ .../src/node/reporters/benchmark/index.ts | 2 + 3 files changed, 37 insertions(+), 132 deletions(-) diff --git a/examples/basic/vite.config.ts b/examples/basic/vite.config.ts index d2f2cf6b8464..8bc6d4f1f0b8 100644 --- a/examples/basic/vite.config.ts +++ b/examples/basic/vite.config.ts @@ -2,108 +2,11 @@ // Configure Vitest (https://vitest.dev/config/) -import fs from 'node:fs' -import path from 'node:path' import { defineConfig } from 'vite' -import { Reporter } from 'vitest/reporters' -import type { BenchmarkResult, File, Task } from 'vitest' - -/* - -VITEST_BENCH_OUTPUT_FILE=main.json pnpm -C examples/basic test bench -- --run -VITEST_BENCH_COMPARE=main.json pnpm -C examples/basic test bench -- --run - -*/ - -type BenchEntries = { [id: string]: Omit }; - -function traverseTask( - task: Task, - callback: (task: Task, depth: number) => void, - depth = 0, -) { - if (task.type === "suite") { - callback(task, depth); - for (const t of task.tasks) { - traverseTask(t, callback, depth + 1); - } - } else { - callback(task, depth); - } -} - -export class CompareReporter implements Reporter { - async onFinished(files: File[] = []) { - // TODO: use env var as flag for prototype - // --benchmark.outputFile - const newFile = process.env.VITEST_BENCH_OUTPUT_FILE; - // --compare - const baseFile = process.env.VITEST_BENCH_COMPARE; - - let baseEntries: BenchEntries | undefined; - if (baseFile) { - if (fs.existsSync(baseFile)) { - baseEntries = JSON.parse(await fs.promises.readFile(baseFile, "utf-8")); - } else { - console.error('baseline not found:', baseFile) - } - } - - if (!newFile && !baseFile) { - console.log("use --benchmark.outputFile to save current run "); - return; - } - - if (baseFile) { - console.log('') - console.log('[BENCH] baseline:', path.resolve(baseFile)); - console.log('') - } - - const currentEntries: BenchEntries = {} - for (const file of files) { - traverseTask(file, (t, depth) => { - if (t.type === "suite") { - console.log(" ".repeat(depth * 2), t.name); - return; - } - if (t.result?.benchmark) { - const { samples, ...current } = t.result.benchmark - currentEntries[t.id] = current; - - const baseline = baseEntries?.[t.id]; - if (baseline) { - const diff = current.hz / baseline.hz; - console.log( - " ".repeat(depth * 2), - t.name, - `${current.hz.toPrecision(5)}hz`, - `[baseline: ${baseline.hz.toPrecision(5)}hz]`, - `[change: ${diff.toFixed(2)}x ${diff > 1 ? '⇑' : '⇓'}]`, - ) - } else { - console.log( - " ".repeat(depth * 2), - `${current.hz.toPrecision(5)}hz`, - `(no baseline)`, - ) - } - } - }) - } - - if (newFile) { - if (!fs.existsSync(path.dirname(newFile))) - await fs.promises.mkdir(path.dirname(newFile), { recursive: true }) - await fs.promises.writeFile(newFile, JSON.stringify(currentEntries, null, 2)); - } - } -} export default defineConfig({ test: { - benchmark: { - reporters: ['default', new CompareReporter()], - }, + /* for example, use global to avoid globals imports (describe, test, expect): */ + // globals: true, }, }) diff --git a/packages/vitest/src/node/reporters/benchmark/compare.ts b/packages/vitest/src/node/reporters/benchmark/compare.ts index 083ab566b5da..fbb838444993 100644 --- a/packages/vitest/src/node/reporters/benchmark/compare.ts +++ b/packages/vitest/src/node/reporters/benchmark/compare.ts @@ -4,25 +4,25 @@ import type { BenchmarkResult, File, Reporter, Task, Vitest } from '../../../typ /* -VITEST_BENCH_OUTPUT_FILE=main.json pnpm -C examples/basic test bench -- --run -VITEST_BENCH_COMPARE=main.json pnpm -C examples/basic test bench -- --run +VITEST_BENCH_OUTPUT_FILE=main.json pnpm -C examples/basic test bench -- --run --reporter default --reporter wip +VITEST_BENCH_COMPARE=main.json pnpm -C examples/basic test bench -- --reporter default --reporter wip */ -type BenchEntries = { [id: string]: Omit }; +interface BenchEntries { [id: string]: Omit } function traverseTask( task: Task, callback: (task: Task, depth: number) => void, depth = 0, ) { - if (task.type === "suite") { - callback(task, depth); - for (const t of task.tasks) { - traverseTask(t, callback, depth + 1); - } - } else { - callback(task, depth); + if (task.type === 'suite') { + callback(task, depth) + for (const t of task.tasks) + traverseTask(t, callback, depth + 1) + } + else { + callback(task, depth) } } @@ -36,54 +36,54 @@ export class CompareReporter implements Reporter { async onFinished(files: File[] = []) { // TODO: use env var as flag for prototype // --benchmark.outputFile - const newFile = process.env.VITEST_BENCH_OUTPUT_FILE; + const newFile = process.env.VITEST_BENCH_OUTPUT_FILE // --compare - const baseFile = process.env.VITEST_BENCH_COMPARE; + const baseFile = process.env.VITEST_BENCH_COMPARE - let baseEntries: BenchEntries | undefined; + let baseEntries: BenchEntries | undefined if (baseFile) { - if (fs.existsSync(baseFile)) { - baseEntries = JSON.parse(await fs.promises.readFile(baseFile, "utf-8")); - } else { - console.error('baseline not found:', baseFile) - } + if (fs.existsSync(baseFile)) + baseEntries = JSON.parse(await fs.promises.readFile(baseFile, 'utf-8')) + else + this.ctx.logger.log('baseline not found:', baseFile) } if (!newFile && !baseFile) { - console.log("use --benchmark.outputFile to save current run "); - return; + this.ctx.logger.log('Use --benchmark.outputFile to save current run ') + return } if (baseFile) { - console.log('') - console.log('[BENCH] baseline:', path.resolve(baseFile)); - console.log('') + this.ctx.logger.log('') + this.ctx.logger.log('[BENCH] baseline:', path.resolve(baseFile)) + this.ctx.logger.log('') } const currentEntries: BenchEntries = {} for (const file of files) { traverseTask(file, (t, depth) => { - if (t.type === "suite") { - this.ctx.logger.log(" ".repeat(depth * 2), t.name); - return; + if (t.type === 'suite') { + this.ctx.logger.log(' '.repeat(depth * 2), t.name) + return } if (t.result?.benchmark) { const { samples: _samples, ...current } = t.result.benchmark - currentEntries[t.id] = current; + currentEntries[t.id] = current - const baseline = baseEntries?.[t.id]; + const baseline = baseEntries?.[t.id] if (baseline) { - const diff = current.hz / baseline.hz; + const diff = current.hz / baseline.hz this.ctx.logger.log( - " ".repeat(depth * 2), + ' '.repeat(depth * 2), t.name, `${current.hz.toPrecision(5)}hz`, `[baseline: ${baseline.hz.toPrecision(5)}hz]`, `[change: ${diff.toFixed(2)}x ${diff > 1 ? '⇑' : '⇓'}]`, ) - } else { + } + else { this.ctx.logger.log( - " ".repeat(depth * 2), + ' '.repeat(depth * 2), `${current.hz.toPrecision(5)}hz`, `(no baseline)`, ) @@ -95,7 +95,7 @@ export class CompareReporter implements Reporter { if (newFile) { if (!fs.existsSync(path.dirname(newFile))) await fs.promises.mkdir(path.dirname(newFile), { recursive: true }) - await fs.promises.writeFile(newFile, JSON.stringify(currentEntries, null, 2)); + await fs.promises.writeFile(newFile, JSON.stringify(currentEntries, null, 2)) } } } diff --git a/packages/vitest/src/node/reporters/benchmark/index.ts b/packages/vitest/src/node/reporters/benchmark/index.ts index 00aa184557b4..3469d23aa8fa 100644 --- a/packages/vitest/src/node/reporters/benchmark/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/index.ts @@ -1,4 +1,5 @@ import { VerboseReporter } from '../verbose' +import { CompareReporter } from './compare' import { JsonReporter } from './json' import { TableReporter } from './table' @@ -6,5 +7,6 @@ export const BenchmarkReportsMap = { default: TableReporter, verbose: VerboseReporter, json: JsonReporter, + wip: CompareReporter, } export type BenchmarkBuiltinReporters = keyof typeof BenchmarkReportsMap From 51941606da816839b4b01ec686dc1e0e23a61d9a Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 16:51:12 +0900 Subject: [PATCH 05/33] wip: mockup --- .../reporters/benchmark/table/tableRender.ts | 165 +++++++++++++++++- 1 file changed, 158 insertions(+), 7 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index ee7b17c48be3..d0f0e9aa4301 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -60,7 +60,8 @@ function renderBenchmarkItems(result: BenchmarkResult) { formatNumber(result.p995 || 0), formatNumber(result.p999 || 0), `±${(result.rme || 0).toFixed(2)}%`, - result.samples.length.toString(), + // TODO: persist only sampleCount? + result.samples ? result.samples.length.toString() : "-", ] } function renderBenchmark(task: Benchmark, tasks: Task[]): string { @@ -92,6 +93,8 @@ function renderBenchmark(task: Benchmark, tasks: Task[]): string { c.cyan(padded[8]), // p999 c.dim(padded[9]), // rem c.dim(padded[10]), // sample + // TODO: fix or hide fastest/slowest during compare + // show "change: 0.98x ⇓" compared to baseline result.rank === 1 ? c.bold(c.green(' fastest')) : (result.rank === benches.length && benches.length > 2) @@ -100,15 +103,134 @@ function renderBenchmark(task: Benchmark, tasks: Task[]): string { ].join(' ') } +// TODO: load from --compare +const baselineResults = { + "-307977539_0_0": { + "name": "normal", + "rank": 1, + "rme": 1.1364415465735003, + "totalTime": 513.9135180000001, + "min": 14.029334999999946, + "max": 15.713413999999943, + "hz": 58.1048440527692, + "period": 14.683243371428574, + "mean": 14.683243371428574, + "variance": 0.23565419608982463, + "sd": 0.4854422685447, + "sem": 0.08205471973712326, + "df": 34, + "critical": 2.0336, + "moe": 0.16686647805741386, + "p75": 15.116964999999936, + "p99": 15.713413999999943, + "p995": 15.713413999999943, + "p999": 15.713413999999943 + }, + "-307977539_0_1": { + "name": "reverse", + "rank": 2, + "rme": 25.34288206604909, + "totalTime": 532.4783659999998, + "min": 17.00932499999999, + "max": 81.15657699999997, + "hz": 22.536126848015464, + "period": 44.37319716666665, + "mean": 44.37319716666665, + "variance": 313.25254577674025, + "sd": 17.698941939470288, + "sem": 5.109244446562364, + "df": 11, + "critical": 2.201, + "moe": 11.245447026883765, + "p75": 50.36103299999991, + "p99": 81.15657699999997, + "p995": 81.15657699999997, + "p999": 81.15657699999997 + } +} as any as Record; + +function computeColumnWidths(results: BenchmarkResult[]): number[] { + const rows = [ + tableHead, + ...results.map(v => renderBenchmarkItems(v)), + ] + return Array.from( + tableHead, + (_, i) => Math.max(...rows.map(row => stripAnsi(row[i]).length)) + ) +} + +function padRow(row: string[], widths: number[]) { + return row.map((v, i) => + i + ? v.padStart(widths[i], ' ') + : v.padEnd(widths[i], ' ') // name + ) +} + +function renderTableHead2(widths: number[]) { + return " ".repeat(3) + padRow(tableHead, widths).map(c.bold).join(' ') +} + +function renderBenchmark2(result: BenchmarkResult, widths: number[]) { + const padded = padRow(renderBenchmarkItems(result), widths); + return [ + padded[0], // name + c.blue(padded[1]), // hz + c.cyan(padded[2]), // min + c.cyan(padded[3]), // max + c.cyan(padded[4]), // mean + c.cyan(padded[5]), // p75 + c.cyan(padded[6]), // p99 + c.cyan(padded[7]), // p995 + c.cyan(padded[8]), // p999 + c.dim(padded[9]), // rem + c.dim(padded[10]), // sample + // TODO: fix or hide fastest/slowest during compare + // show "change: 0.98x ⇓" compared to baseline + // result.rank === 1 + // ? c.bold(c.green(' fastest')) + // : '', + // result.rank === 1 + // ? c.bold(c.green(' fastest')) + // : (result.rank > 2 && result.rank === benches.length && benches.length > 2) + // ? c.bold(c.gray(' slowest')) + // : '', + ].join(' ') +} + function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): string { const output: string[] = [] + const benchMap: Record = {}; + for (const t of tasks) { + if (t.meta.benchmark && t.result?.benchmark) { + benchMap[t.id] = { + current: t.result.benchmark, + baseline: { + ...baselineResults[t.id], + name: c.gray("(prev.)") + } + } + } + } + const benchCount = Object.entries(benchMap).length; + + // compute column widths + const columnWidths = computeColumnWidths( + Object.values(benchMap) + .flatMap(v => [v.current, v.baseline]) + .filter(notNullish) + ) + let idx = 0 for (const task of tasks) { const padding = ' '.repeat(level ? 1 : 0) let prefix = '' + // if (idx === 0 && task.meta?.benchmark) + // prefix += `${renderTableHead(tasks)}\n${padding}` if (idx === 0 && task.meta?.benchmark) - prefix += `${renderTableHead(tasks)}\n${padding}` + prefix += `${renderTableHead2(columnWidths)}\n${padding}` prefix += ` ${getStateSymbol(task)} ` @@ -131,11 +253,40 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st if (level === 0) name = formatFilepath(name) - const body = task.meta?.benchmark - ? renderBenchmark(task as Benchmark, tasks) - : name - - output.push(padding + prefix + body + suffix) + const bench = benchMap[task.id] + if (bench) { + let body = renderBenchmark2(bench.current, columnWidths); + if (bench.baseline) { + if (bench.current.hz) { + const diff = bench.current.hz / bench.baseline.hz; + const diffFixed = diff.toFixed(2); + if (diffFixed === "1.0.0") { + body += ` ` + c.gray(`[${diffFixed}x]`) + } if (diff > 1) { + body += ` ` + c.blue(`[${diffFixed}x] ⇑`) + } else { + body += ` ` + c.red(`[${diffFixed}x] ⇓`) + } + } + output.push(padding + prefix + body + suffix) + const body2 = renderBenchmark2(bench.baseline, columnWidths); + output.push(padding + ` ` + body2 + suffix) + } else { + if (bench.current.rank === 1) { + body += ` ` + c.bold(c.green(' fastest')); + } + if (bench.current.rank === benchCount && benchCount > 2) { + body += ` ` + c.bold(c.gray(' slowest')); + } + output.push(padding + prefix + body + suffix) + } + } else { + output.push(padding + prefix + name + suffix); + } + // // const body = task.meta?.benchmark + // // ? renderBenchmark(task as Benchmark, tasks) + // // : name + // output.push(padding + prefix + name + suffix) if ((task.result?.state !== 'pass') && outputMap.get(task) != null) { let data: string | undefined = outputMap.get(task) From 5219d0d864ff00a180a013381fce91b891baed15 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 18:10:35 +0900 Subject: [PATCH 06/33] wip: support --compare --- packages/vitest/src/defaults.ts | 2 +- packages/vitest/src/node/cli/cli-config.ts | 4 ++ packages/vitest/src/node/config.ts | 4 ++ .../node/reporters/benchmark/table/index.ts | 44 ++++++++++++++++++- .../reporters/benchmark/table/tableRender.ts | 26 +++++------ packages/vitest/src/types/benchmark.ts | 5 +++ packages/vitest/src/types/config.ts | 7 ++- 7 files changed, 74 insertions(+), 18 deletions(-) diff --git a/packages/vitest/src/defaults.ts b/packages/vitest/src/defaults.ts index 44d21c4a350f..d385817d3441 100644 --- a/packages/vitest/src/defaults.ts +++ b/packages/vitest/src/defaults.ts @@ -4,7 +4,7 @@ import { isCI } from './utils/env' export const defaultInclude = ['**/*.{test,spec}.?(c|m)[jt]s?(x)'] export const defaultExclude = ['**/node_modules/**', '**/dist/**', '**/cypress/**', '**/.{idea,git,cache,output,temp}/**', '**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*'] -export const benchmarkConfigDefaults: Required> = { +export const benchmarkConfigDefaults: Required> = { include: ['**/*.{bench,benchmark}.?(c|m)[jt]s?(x)'], exclude: defaultExclude, includeSource: [], diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index b5e16b9f526c..120d572dfdca 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -582,6 +582,10 @@ export const cliOptionsConfig: VitestCLIOptions = { clearScreen: { description: 'Clear terminal screen when re-running tests during watch mode (default: true)', }, + compare: { + description: 'benchmark output file to compare against', + argument: '', + }, // disable CLI options cliExclude: null, diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 9d1c96a18fa3..2fc9ef7f27c9 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -348,6 +348,10 @@ export function resolveConfig( if (options.outputFile) resolved.benchmark.outputFile = options.outputFile + + // --compare from cli + if (options.compare) + resolved.benchmark.compare = options.compare } resolved.setupFiles = toArray(resolved.setupFiles || []).map(file => diff --git a/packages/vitest/src/node/reporters/benchmark/table/index.ts b/packages/vitest/src/node/reporters/benchmark/table/index.ts index 751e2431717a..e7715f7aec50 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/index.ts @@ -2,6 +2,11 @@ import c from 'picocolors' import type { UserConsoleLog } from '../../../../types/general' import { BaseReporter } from '../../base' import { type TableRendererOptions, createTableRenderer } from './tableRender' +import type { BenchmarkResult, File } from '../../../../types' +import { getTasks } from '../../../../utils' +import { getOutputFile } from '../../../../utils/config-helpers' +import * as pathe from 'pathe' +import fs from 'node:fs' export class TableReporter extends BaseReporter { renderer?: ReturnType @@ -17,11 +22,21 @@ export class TableReporter extends BaseReporter { super.onWatcherStart() } - onCollected() { + async onCollected() { if (this.isTTY) { this.rendererOptions.logger = this.ctx.logger this.rendererOptions.showHeap = this.ctx.config.logHeapUsage this.rendererOptions.slowTestThreshold = this.ctx.config.slowTestThreshold + if (this.ctx.config.benchmark?.compare) { + const compareFile = pathe.resolve(this.ctx.config.root, this.ctx.config.benchmark?.compare) + try { + this.rendererOptions.compare = JSON.parse( + await fs.promises.readFile(compareFile, "utf-8") + ) + } catch (e) { + this.ctx.logger.error(`Failed to read '${compareFile}'`, e); + } + } const files = this.ctx.state.getFiles(this.watchFilters) if (!this.renderer) this.renderer = createTableRenderer(files, this.rendererOptions).start() @@ -34,6 +49,17 @@ export class TableReporter extends BaseReporter { await this.stopListRender() this.ctx.logger.log() await super.onFinished(files, errors) + + // write output for future comparison + let outputFile = getOutputFile(this.ctx.config.benchmark, 'default') + if (outputFile) { + outputFile = pathe.resolve(this.ctx.config.root, outputFile) + const outputDirectory = pathe.dirname(outputFile) + if (!fs.existsSync(outputDirectory)) + await fs.promises.mkdir(outputDirectory, { recursive: true }) + const output = createBenchmarkOutput(files); + await fs.promises.writeFile(outputFile, JSON.stringify(output, null, 2)) + } } async onWatcherStart() { @@ -58,3 +84,19 @@ export class TableReporter extends BaseReporter { super.onUserConsoleLog(log) } } + +export interface TableBenchmarkOutput { + [id: string]: Omit +} + +function createBenchmarkOutput(files: File[]) { + const result: TableBenchmarkOutput = {} + for (const test of getTasks(files)) { + if (test.meta?.benchmark && test.result?.benchmark) { + // strip gigantic "samples" + const { samples: _samples, ...rest } = test.result.benchmark; + result[test.id] = rest; + } + } + return result; +} diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index d0f0e9aa4301..3fd708ed15ec 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -6,12 +6,14 @@ import { getTests, notNullish } from '../../../../utils' import { F_RIGHT } from '../../../../utils/figures' import type { Logger } from '../../../logger' import { getCols, getStateSymbol } from '../../renderers/utils' +import { TableBenchmarkOutput } from '.' export interface TableRendererOptions { renderSucceed?: boolean logger: Logger showHeap: boolean slowTestThreshold: number + compare?: TableBenchmarkOutput; } const outputMap = new WeakMap() @@ -61,7 +63,7 @@ function renderBenchmarkItems(result: BenchmarkResult) { formatNumber(result.p999 || 0), `±${(result.rme || 0).toFixed(2)}%`, // TODO: persist only sampleCount? - result.samples ? result.samples.length.toString() : "-", + result.samples.length ? result.samples.length.toString() : "-", ] } function renderBenchmark(task: Benchmark, tasks: Task[]): string { @@ -186,16 +188,6 @@ function renderBenchmark2(result: BenchmarkResult, widths: number[]) { c.cyan(padded[8]), // p999 c.dim(padded[9]), // rem c.dim(padded[10]), // sample - // TODO: fix or hide fastest/slowest during compare - // show "change: 0.98x ⇓" compared to baseline - // result.rank === 1 - // ? c.bold(c.green(' fastest')) - // : '', - // result.rank === 1 - // ? c.bold(c.green(' fastest')) - // : (result.rank > 2 && result.rank === benches.length && benches.length > 2) - // ? c.bold(c.gray(' slowest')) - // : '', ].join(' ') } @@ -207,10 +199,14 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st if (t.meta.benchmark && t.result?.benchmark) { benchMap[t.id] = { current: t.result.benchmark, - baseline: { - ...baselineResults[t.id], - name: c.gray("(prev.)") + } + if (options.compare && options.compare[t.id]) { + benchMap[t.id].baseline = { + ...options.compare[t.id], + samples: [], + name: c.gray("(prev.)"), } + options.compare[t.id] } } } @@ -256,7 +252,7 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st const bench = benchMap[task.id] if (bench) { let body = renderBenchmark2(bench.current, columnWidths); - if (bench.baseline) { + if (options.compare && bench.baseline) { if (bench.current.hz) { const diff = bench.current.hz / bench.baseline.hz; const diffFixed = diff.toFixed(2); diff --git a/packages/vitest/src/types/benchmark.ts b/packages/vitest/src/types/benchmark.ts index addb0bc71b60..0ff187c7ac85 100644 --- a/packages/vitest/src/types/benchmark.ts +++ b/packages/vitest/src/types/benchmark.ts @@ -39,6 +39,11 @@ export interface BenchmarkUserOptions { * Also definable individually per reporter by using an object instead. */ outputFile?: string | (Partial> & Record) + + /** + * benchmark output file to compare against + */ + compare?: string; } export interface Benchmark extends Custom { diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index f75c660558c1..a68a31689426 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -824,6 +824,11 @@ export interface UserConfig extends InlineConfig { * Override vite config's clearScreen from cli */ clearScreen?: boolean + + /** + * benchmark.compare option exposed at the top level for cli + */ + compare?: string; } export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'browser' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'benchmark' | 'shard' | 'cache' | 'sequence' | 'typecheck' | 'runner' | 'poolOptions' | 'pool' | 'cliExclude'> { @@ -850,7 +855,7 @@ export interface ResolvedConfig extends Omit, 'config' | 'f api?: ApiConfig cliExclude?: string[] - benchmark?: Required> & Pick + benchmark?: Required> & Pick shard?: { index: number count: number From 22c3b4d7716e7e9310d6ca063e2728602abafe57 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 18:11:27 +0900 Subject: [PATCH 07/33] chore: remove unused --- .../reporters/benchmark/table/tableRender.ts | 167 ++++-------------- 1 file changed, 35 insertions(+), 132 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index 3fd708ed15ec..62b8df158227 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -1,19 +1,19 @@ import c from 'picocolors' import cliTruncate from 'cli-truncate' import stripAnsi from 'strip-ansi' -import type { Benchmark, BenchmarkResult, Task } from '../../../../types' +import type { BenchmarkResult, Task } from '../../../../types' import { getTests, notNullish } from '../../../../utils' import { F_RIGHT } from '../../../../utils/figures' import type { Logger } from '../../../logger' import { getCols, getStateSymbol } from '../../renderers/utils' -import { TableBenchmarkOutput } from '.' +import type { TableBenchmarkOutput } from '.' export interface TableRendererOptions { renderSucceed?: boolean logger: Logger showHeap: boolean slowTestThreshold: number - compare?: TableBenchmarkOutput; + compare?: TableBenchmarkOutput } const outputMap = new WeakMap() @@ -37,19 +37,6 @@ function formatNumber(number: number) { const tableHead = ['name', 'hz', 'min', 'max', 'mean', 'p75', 'p99', 'p995', 'p999', 'rme', 'samples'] -function renderTableHead(tasks: Task[]) { - const benches = tasks - .map(i => i.meta?.benchmark ? i.result?.benchmark : undefined) - .filter(notNullish) - const allItems = benches.map(renderBenchmarkItems).concat([tableHead]) - return `${' '.repeat(3)}${tableHead.map((i, idx) => { - const width = Math.max(...allItems.map(i => i[idx].length)) - return idx - ? i.padStart(width, ' ') - : i.padEnd(width, ' ') // name - }).map(c.bold).join(' ')}` -} - function renderBenchmarkItems(result: BenchmarkResult) { return [ result.name, @@ -63,93 +50,9 @@ function renderBenchmarkItems(result: BenchmarkResult) { formatNumber(result.p999 || 0), `±${(result.rme || 0).toFixed(2)}%`, // TODO: persist only sampleCount? - result.samples.length ? result.samples.length.toString() : "-", + result.samples.length ? result.samples.length.toString() : '-', ] } -function renderBenchmark(task: Benchmark, tasks: Task[]): string { - const result = task.result?.benchmark - if (!result) - return task.name - - const benches = tasks - .map(i => i.meta?.benchmark ? i.result?.benchmark : undefined) - .filter(notNullish) - const allItems = benches.map(renderBenchmarkItems).concat([tableHead]) - const items = renderBenchmarkItems(result) - const padded = items.map((i, idx) => { - const width = Math.max(...allItems.map(i => i[idx].length)) - return idx - ? i.padStart(width, ' ') - : i.padEnd(width, ' ') // name - }) - - return [ - padded[0], // name - c.blue(padded[1]), // hz - c.cyan(padded[2]), // min - c.cyan(padded[3]), // max - c.cyan(padded[4]), // mean - c.cyan(padded[5]), // p75 - c.cyan(padded[6]), // p99 - c.cyan(padded[7]), // p995 - c.cyan(padded[8]), // p999 - c.dim(padded[9]), // rem - c.dim(padded[10]), // sample - // TODO: fix or hide fastest/slowest during compare - // show "change: 0.98x ⇓" compared to baseline - result.rank === 1 - ? c.bold(c.green(' fastest')) - : (result.rank === benches.length && benches.length > 2) - ? c.bold(c.gray(' slowest')) - : '', - ].join(' ') -} - -// TODO: load from --compare -const baselineResults = { - "-307977539_0_0": { - "name": "normal", - "rank": 1, - "rme": 1.1364415465735003, - "totalTime": 513.9135180000001, - "min": 14.029334999999946, - "max": 15.713413999999943, - "hz": 58.1048440527692, - "period": 14.683243371428574, - "mean": 14.683243371428574, - "variance": 0.23565419608982463, - "sd": 0.4854422685447, - "sem": 0.08205471973712326, - "df": 34, - "critical": 2.0336, - "moe": 0.16686647805741386, - "p75": 15.116964999999936, - "p99": 15.713413999999943, - "p995": 15.713413999999943, - "p999": 15.713413999999943 - }, - "-307977539_0_1": { - "name": "reverse", - "rank": 2, - "rme": 25.34288206604909, - "totalTime": 532.4783659999998, - "min": 17.00932499999999, - "max": 81.15657699999997, - "hz": 22.536126848015464, - "period": 44.37319716666665, - "mean": 44.37319716666665, - "variance": 313.25254577674025, - "sd": 17.698941939470288, - "sem": 5.109244446562364, - "df": 11, - "critical": 2.201, - "moe": 11.245447026883765, - "p75": 50.36103299999991, - "p99": 81.15657699999997, - "p995": 81.15657699999997, - "p999": 81.15657699999997 - } -} as any as Record; function computeColumnWidths(results: BenchmarkResult[]): number[] { const rows = [ @@ -158,24 +61,24 @@ function computeColumnWidths(results: BenchmarkResult[]): number[] { ] return Array.from( tableHead, - (_, i) => Math.max(...rows.map(row => stripAnsi(row[i]).length)) + (_, i) => Math.max(...rows.map(row => stripAnsi(row[i]).length)), ) } function padRow(row: string[], widths: number[]) { return row.map((v, i) => i - ? v.padStart(widths[i], ' ') - : v.padEnd(widths[i], ' ') // name + ? v.padStart(widths[i], ' ') + : v.padEnd(widths[i], ' '), // name ) } function renderTableHead2(widths: number[]) { - return " ".repeat(3) + padRow(tableHead, widths).map(c.bold).join(' ') + return ' '.repeat(3) + padRow(tableHead, widths).map(c.bold).join(' ') } function renderBenchmark2(result: BenchmarkResult, widths: number[]) { - const padded = padRow(renderBenchmarkItems(result), widths); + const padded = padRow(renderBenchmarkItems(result), widths) return [ padded[0], // name c.blue(padded[1]), // hz @@ -194,7 +97,7 @@ function renderBenchmark2(result: BenchmarkResult, widths: number[]) { function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): string { const output: string[] = [] - const benchMap: Record = {}; + const benchMap: Record = {} for (const t of tasks) { if (t.meta.benchmark && t.result?.benchmark) { benchMap[t.id] = { @@ -204,19 +107,18 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st benchMap[t.id].baseline = { ...options.compare[t.id], samples: [], - name: c.gray("(prev.)"), + name: c.gray('(prev.)'), } - options.compare[t.id] } } } - const benchCount = Object.entries(benchMap).length; + const benchCount = Object.entries(benchMap).length // compute column widths const columnWidths = computeColumnWidths( Object.values(benchMap) .flatMap(v => [v.current, v.baseline]) - .filter(notNullish) + .filter(notNullish), ) let idx = 0 @@ -251,33 +153,34 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st const bench = benchMap[task.id] if (bench) { - let body = renderBenchmark2(bench.current, columnWidths); + let body = renderBenchmark2(bench.current, columnWidths) if (options.compare && bench.baseline) { if (bench.current.hz) { - const diff = bench.current.hz / bench.baseline.hz; - const diffFixed = diff.toFixed(2); - if (diffFixed === "1.0.0") { - body += ` ` + c.gray(`[${diffFixed}x]`) - } if (diff > 1) { - body += ` ` + c.blue(`[${diffFixed}x] ⇑`) - } else { - body += ` ` + c.red(`[${diffFixed}x] ⇓`) - } + const diff = bench.current.hz / bench.baseline.hz + const diffFixed = diff.toFixed(2) + if (diffFixed === '1.0.0') + body += ` ${c.gray(`[${diffFixed}x]`)}` + if (diff > 1) + body += ` ${c.blue(`[${diffFixed}x] ⇑`)}` + else + body += ` ${c.red(`[${diffFixed}x] ⇓`)}` } output.push(padding + prefix + body + suffix) - const body2 = renderBenchmark2(bench.baseline, columnWidths); - output.push(padding + ` ` + body2 + suffix) - } else { - if (bench.current.rank === 1) { - body += ` ` + c.bold(c.green(' fastest')); - } - if (bench.current.rank === benchCount && benchCount > 2) { - body += ` ` + c.bold(c.gray(' slowest')); - } + const body2 = renderBenchmark2(bench.baseline, columnWidths) + output.push(`${padding} ${body2}${suffix}`) + } + else { + if (bench.current.rank === 1) + body += ` ${c.bold(c.green(' fastest'))}` + + if (bench.current.rank === benchCount && benchCount > 2) + body += ` ${c.bold(c.gray(' slowest'))}` + output.push(padding + prefix + body + suffix) } - } else { - output.push(padding + prefix + name + suffix); + } + else { + output.push(padding + prefix + name + suffix) } // // const body = task.meta?.benchmark // // ? renderBenchmark(task as Benchmark, tasks) From 0a23d81b22a6218568ffb5ee33bb4ead584bae07 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 18:12:51 +0900 Subject: [PATCH 08/33] chore: cleanup --- .../reporters/benchmark/table/tableRender.ts | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index 62b8df158227..1caee38ccc5b 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -73,11 +73,11 @@ function padRow(row: string[], widths: number[]) { ) } -function renderTableHead2(widths: number[]) { +function renderTableHead(widths: number[]) { return ' '.repeat(3) + padRow(tableHead, widths).map(c.bold).join(' ') } -function renderBenchmark2(result: BenchmarkResult, widths: number[]) { +function renderBenchmark(result: BenchmarkResult, widths: number[]) { const padded = padRow(renderBenchmarkItems(result), widths) return [ padded[0], // name @@ -125,10 +125,8 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st for (const task of tasks) { const padding = ' '.repeat(level ? 1 : 0) let prefix = '' - // if (idx === 0 && task.meta?.benchmark) - // prefix += `${renderTableHead(tasks)}\n${padding}` if (idx === 0 && task.meta?.benchmark) - prefix += `${renderTableHead2(columnWidths)}\n${padding}` + prefix += `${renderTableHead(columnWidths)}\n${padding}` prefix += ` ${getStateSymbol(task)} ` @@ -153,7 +151,7 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st const bench = benchMap[task.id] if (bench) { - let body = renderBenchmark2(bench.current, columnWidths) + let body = renderBenchmark(bench.current, columnWidths) if (options.compare && bench.baseline) { if (bench.current.hz) { const diff = bench.current.hz / bench.baseline.hz @@ -166,11 +164,11 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st body += ` ${c.red(`[${diffFixed}x] ⇓`)}` } output.push(padding + prefix + body + suffix) - const body2 = renderBenchmark2(bench.baseline, columnWidths) - output.push(`${padding} ${body2}${suffix}`) + const bodyBaseline = renderBenchmark(bench.baseline, columnWidths) + output.push(`${padding} ${bodyBaseline}${suffix}`) } else { - if (bench.current.rank === 1) + if (bench.current.rank === 1 && benchCount > 1) body += ` ${c.bold(c.green(' fastest'))}` if (bench.current.rank === benchCount && benchCount > 2) @@ -182,10 +180,6 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st else { output.push(padding + prefix + name + suffix) } - // // const body = task.meta?.benchmark - // // ? renderBenchmark(task as Benchmark, tasks) - // // : name - // output.push(padding + prefix + name + suffix) if ((task.result?.state !== 'pass') && outputMap.get(task) != null) { let data: string | undefined = outputMap.get(task) From 1658a2d8ec8de2cdf44eb5c39bb647dff6311f78 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 18:14:09 +0900 Subject: [PATCH 09/33] chore: unused --- .../src/node/reporters/benchmark/compare.ts | 101 ------------------ .../src/node/reporters/benchmark/index.ts | 2 - 2 files changed, 103 deletions(-) delete mode 100644 packages/vitest/src/node/reporters/benchmark/compare.ts diff --git a/packages/vitest/src/node/reporters/benchmark/compare.ts b/packages/vitest/src/node/reporters/benchmark/compare.ts deleted file mode 100644 index fbb838444993..000000000000 --- a/packages/vitest/src/node/reporters/benchmark/compare.ts +++ /dev/null @@ -1,101 +0,0 @@ -import fs from 'node:fs' -import path from 'node:path' -import type { BenchmarkResult, File, Reporter, Task, Vitest } from '../../../types' - -/* - -VITEST_BENCH_OUTPUT_FILE=main.json pnpm -C examples/basic test bench -- --run --reporter default --reporter wip -VITEST_BENCH_COMPARE=main.json pnpm -C examples/basic test bench -- --reporter default --reporter wip - -*/ - -interface BenchEntries { [id: string]: Omit } - -function traverseTask( - task: Task, - callback: (task: Task, depth: number) => void, - depth = 0, -) { - if (task.type === 'suite') { - callback(task, depth) - for (const t of task.tasks) - traverseTask(t, callback, depth + 1) - } - else { - callback(task, depth) - } -} - -export class CompareReporter implements Reporter { - ctx: Vitest = undefined! - - onInit(ctx: Vitest) { - this.ctx = ctx - } - - async onFinished(files: File[] = []) { - // TODO: use env var as flag for prototype - // --benchmark.outputFile - const newFile = process.env.VITEST_BENCH_OUTPUT_FILE - // --compare - const baseFile = process.env.VITEST_BENCH_COMPARE - - let baseEntries: BenchEntries | undefined - if (baseFile) { - if (fs.existsSync(baseFile)) - baseEntries = JSON.parse(await fs.promises.readFile(baseFile, 'utf-8')) - else - this.ctx.logger.log('baseline not found:', baseFile) - } - - if (!newFile && !baseFile) { - this.ctx.logger.log('Use --benchmark.outputFile to save current run ') - return - } - - if (baseFile) { - this.ctx.logger.log('') - this.ctx.logger.log('[BENCH] baseline:', path.resolve(baseFile)) - this.ctx.logger.log('') - } - - const currentEntries: BenchEntries = {} - for (const file of files) { - traverseTask(file, (t, depth) => { - if (t.type === 'suite') { - this.ctx.logger.log(' '.repeat(depth * 2), t.name) - return - } - if (t.result?.benchmark) { - const { samples: _samples, ...current } = t.result.benchmark - currentEntries[t.id] = current - - const baseline = baseEntries?.[t.id] - if (baseline) { - const diff = current.hz / baseline.hz - this.ctx.logger.log( - ' '.repeat(depth * 2), - t.name, - `${current.hz.toPrecision(5)}hz`, - `[baseline: ${baseline.hz.toPrecision(5)}hz]`, - `[change: ${diff.toFixed(2)}x ${diff > 1 ? '⇑' : '⇓'}]`, - ) - } - else { - this.ctx.logger.log( - ' '.repeat(depth * 2), - `${current.hz.toPrecision(5)}hz`, - `(no baseline)`, - ) - } - } - }) - } - - if (newFile) { - if (!fs.existsSync(path.dirname(newFile))) - await fs.promises.mkdir(path.dirname(newFile), { recursive: true }) - await fs.promises.writeFile(newFile, JSON.stringify(currentEntries, null, 2)) - } - } -} diff --git a/packages/vitest/src/node/reporters/benchmark/index.ts b/packages/vitest/src/node/reporters/benchmark/index.ts index 3469d23aa8fa..00aa184557b4 100644 --- a/packages/vitest/src/node/reporters/benchmark/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/index.ts @@ -1,5 +1,4 @@ import { VerboseReporter } from '../verbose' -import { CompareReporter } from './compare' import { JsonReporter } from './json' import { TableReporter } from './table' @@ -7,6 +6,5 @@ export const BenchmarkReportsMap = { default: TableReporter, verbose: VerboseReporter, json: JsonReporter, - wip: CompareReporter, } export type BenchmarkBuiltinReporters = keyof typeof BenchmarkReportsMap From 130ac47efc59190ddd2425efcdcfcdfd31968867 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 18:16:02 +0900 Subject: [PATCH 10/33] chore: tweak --- .../vitest/src/node/reporters/benchmark/table/tableRender.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index 1caee38ccc5b..7cd45cb77f0d 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -107,7 +107,7 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st benchMap[t.id].baseline = { ...options.compare[t.id], samples: [], - name: c.gray('(prev.)'), + name: c.gray('(comp.)'), } } } From 4e4e0062c00ced38af216c446f01e6496b0c6bc2 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 18:22:51 +0900 Subject: [PATCH 11/33] chore: lint --- .../node/reporters/benchmark/table/index.ts | 23 ++++++++++--------- packages/vitest/src/types/benchmark.ts | 2 +- packages/vitest/src/types/config.ts | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/index.ts b/packages/vitest/src/node/reporters/benchmark/table/index.ts index e7715f7aec50..387c6eb8484c 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/index.ts @@ -1,12 +1,12 @@ +import fs from 'node:fs' import c from 'picocolors' +import * as pathe from 'pathe' import type { UserConsoleLog } from '../../../../types/general' import { BaseReporter } from '../../base' -import { type TableRendererOptions, createTableRenderer } from './tableRender' import type { BenchmarkResult, File } from '../../../../types' import { getTasks } from '../../../../utils' import { getOutputFile } from '../../../../utils/config-helpers' -import * as pathe from 'pathe' -import fs from 'node:fs' +import { type TableRendererOptions, createTableRenderer } from './tableRender' export class TableReporter extends BaseReporter { renderer?: ReturnType @@ -31,10 +31,11 @@ export class TableReporter extends BaseReporter { const compareFile = pathe.resolve(this.ctx.config.root, this.ctx.config.benchmark?.compare) try { this.rendererOptions.compare = JSON.parse( - await fs.promises.readFile(compareFile, "utf-8") + await fs.promises.readFile(compareFile, 'utf-8'), ) - } catch (e) { - this.ctx.logger.error(`Failed to read '${compareFile}'`, e); + } + catch (e) { + this.ctx.logger.error(`Failed to read '${compareFile}'`, e) } } const files = this.ctx.state.getFiles(this.watchFilters) @@ -57,7 +58,7 @@ export class TableReporter extends BaseReporter { const outputDirectory = pathe.dirname(outputFile) if (!fs.existsSync(outputDirectory)) await fs.promises.mkdir(outputDirectory, { recursive: true }) - const output = createBenchmarkOutput(files); + const output = createBenchmarkOutput(files) await fs.promises.writeFile(outputFile, JSON.stringify(output, null, 2)) } } @@ -86,7 +87,7 @@ export class TableReporter extends BaseReporter { } export interface TableBenchmarkOutput { - [id: string]: Omit + [id: string]: Omit } function createBenchmarkOutput(files: File[]) { @@ -94,9 +95,9 @@ function createBenchmarkOutput(files: File[]) { for (const test of getTasks(files)) { if (test.meta?.benchmark && test.result?.benchmark) { // strip gigantic "samples" - const { samples: _samples, ...rest } = test.result.benchmark; - result[test.id] = rest; + const { samples: _samples, ...rest } = test.result.benchmark + result[test.id] = rest } } - return result; + return result } diff --git a/packages/vitest/src/types/benchmark.ts b/packages/vitest/src/types/benchmark.ts index 0ff187c7ac85..df232dd05294 100644 --- a/packages/vitest/src/types/benchmark.ts +++ b/packages/vitest/src/types/benchmark.ts @@ -43,7 +43,7 @@ export interface BenchmarkUserOptions { /** * benchmark output file to compare against */ - compare?: string; + compare?: string } export interface Benchmark extends Custom { diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index a68a31689426..c50738dd02ed 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -828,7 +828,7 @@ export interface UserConfig extends InlineConfig { /** * benchmark.compare option exposed at the top level for cli */ - compare?: string; + compare?: string } export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'browser' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'benchmark' | 'shard' | 'cache' | 'sequence' | 'typecheck' | 'runner' | 'poolOptions' | 'pool' | 'cliExclude'> { From c02a0c1bdb53d537405f43b2d8f9bc5e4f0fd8fa Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 19:23:36 +0900 Subject: [PATCH 12/33] test: tweak example --- examples/basic/test/base.bench.ts | 22 ---------------------- test/benchmark/specs/runner.test.mjs | 17 +++++++++++++++++ test/benchmark/vitest.config.ts | 17 ----------------- 3 files changed, 17 insertions(+), 39 deletions(-) delete mode 100644 examples/basic/test/base.bench.ts diff --git a/examples/basic/test/base.bench.ts b/examples/basic/test/base.bench.ts deleted file mode 100644 index e059812d32fc..000000000000 --- a/examples/basic/test/base.bench.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { bench, describe } from 'vitest' - -describe('sort', () => { - let x: number[] - let y: number[] - - bench('normal', () => { - x.sort((a, b) => a - b) - }, { - setup: () => { - x = Array.from({ length: 1000_000 }, (_, i) => i) - }, - }) - - bench('reverse', () => { - y.sort((a, b) => a - b) - }, { - setup: () => { - y = Array.from({ length: 1000_000 }, (_, i) => -i) - }, - }) -}) diff --git a/test/benchmark/specs/runner.test.mjs b/test/benchmark/specs/runner.test.mjs index 0a53b7ab05a4..23ee531e09d8 100644 --- a/test/benchmark/specs/runner.test.mjs +++ b/test/benchmark/specs/runner.test.mjs @@ -8,8 +8,25 @@ if (existsSync('./bench.json')) rmSync('./bench.json') try { + function noop() {} + await startVitest('benchmark', ['base.bench', 'mode.bench', 'only.bench'], { watch: false, + benchmark: { + reporters: ['json', { + onInit: noop, + onPathsCollected: noop, + onCollected: noop, + onFinished: noop, + onTaskUpdate: noop, + onTestRemoved: noop, + onWatcherStart: noop, + onWatcherRerun: noop, + onServerRestart: noop, + onUserConsoleLog: noop, + }], + outputFile: './bench.json', + }, }) } catch (error) { diff --git a/test/benchmark/vitest.config.ts b/test/benchmark/vitest.config.ts index 790c4797b8e3..dd76716d8d79 100644 --- a/test/benchmark/vitest.config.ts +++ b/test/benchmark/vitest.config.ts @@ -1,25 +1,8 @@ import { defineConfig } from 'vitest/config' -function noop() {} - export default defineConfig({ test: { update: false, allowOnly: true, - benchmark: { - outputFile: './bench.json', - reporters: ['json', { - onInit: noop, - onPathsCollected: noop, - onCollected: noop, - onFinished: noop, - onTaskUpdate: noop, - onTestRemoved: noop, - onWatcherStart: noop, - onWatcherRerun: noop, - onServerRestart: noop, - onUserConsoleLog: noop, - }], - }, }, }) From 2943411ea042001eab0305af5ad5200e5e2b657c Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 19 Mar 2024 19:26:37 +0900 Subject: [PATCH 13/33] chore: tweak style --- .../vitest/src/node/reporters/benchmark/table/tableRender.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index 7cd45cb77f0d..b428e0223151 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -107,7 +107,7 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st benchMap[t.id].baseline = { ...options.compare[t.id], samples: [], - name: c.gray('(comp.)'), + name: '', } } } @@ -165,7 +165,7 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st } output.push(padding + prefix + body + suffix) const bodyBaseline = renderBenchmark(bench.baseline, columnWidths) - output.push(`${padding} ${bodyBaseline}${suffix}`) + output.push(`${padding} ${bodyBaseline} ${c.dim('(baseline)')}`) } else { if (bench.current.rank === 1 && benchCount > 1) From 25702bc43b4c2c8860bf0b48a78afa3dbb7c951b Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 20 Mar 2024 10:10:16 +0900 Subject: [PATCH 14/33] test: add test --- pnpm-lock.yaml | 59 ++++++++++--------- .../fixtures/basic/basic.bench.ts | 13 ++++ test/benchmark-wip/fixtures/basic/bench.json | 44 ++++++++++++++ .../fixtures/basic/vitest.config.ts | 3 + test/benchmark-wip/package.json | 11 ++++ test/benchmark-wip/test/compare.test.ts | 49 +++++++++++++++ test/benchmark-wip/vitest.config.ts | 11 ++++ 7 files changed, 163 insertions(+), 27 deletions(-) create mode 100644 test/benchmark-wip/fixtures/basic/basic.bench.ts create mode 100644 test/benchmark-wip/fixtures/basic/bench.json create mode 100644 test/benchmark-wip/fixtures/basic/vitest.config.ts create mode 100644 test/benchmark-wip/package.json create mode 100644 test/benchmark-wip/test/compare.test.ts create mode 100644 test/benchmark-wip/vitest.config.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1ee182d27ca4..0d16075e3eb7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1538,6 +1538,12 @@ importers: specifier: workspace:* version: link:../../packages/vitest + test/benchmark-wip: + devDependencies: + vitest: + specifier: workspace:* + version: link:../../packages/vitest + test/browser: devDependencies: '@vitejs/plugin-basic-ssl': @@ -5810,7 +5816,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 jest-message-util: 27.5.1 jest-util: 27.5.1 @@ -5831,7 +5837,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.8.1 @@ -5868,7 +5874,7 @@ packages: dependencies: '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 jest-mock: 27.5.1 dev: true @@ -5885,7 +5891,7 @@ packages: dependencies: '@jest/types': 27.5.1 '@sinonjs/fake-timers': 8.1.0 - '@types/node': 20.11.27 + '@types/node': 20.11.30 jest-message-util: 27.5.1 jest-mock: 27.5.1 jest-util: 27.5.1 @@ -5914,7 +5920,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 collect-v8-coverage: 1.0.2 exit: 0.1.2 @@ -6045,7 +6051,7 @@ packages: dependencies: '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 20.11.27 + '@types/node': 20.11.30 '@types/yargs': 16.0.7 chalk: 4.1.2 dev: true @@ -9391,7 +9397,7 @@ packages: resolution: {integrity: sha512-N7UDG0/xiPQa2D/XrVJXjkWbpqHCd2sBaB32ggRF2l83RhPfamgKGF8gwwqyksS95qUS5ZYF9aF+lLPRlwI2UA==} dependencies: '@types/connect': 3.4.37 - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/braces@3.0.1: @@ -9412,7 +9418,7 @@ packages: /@types/connect@3.4.37: resolution: {integrity: sha512-zBUSRqkfZ59OcwXon4HVxhx5oWCJmc0OtBTK05M+p0dYjgN6iTwIL2T/WbsQZrEsdnwaF9cWQ+azOnpPvIqY3Q==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/cookie@0.4.1: @@ -9479,7 +9485,7 @@ packages: /@types/express-serve-static-core@4.17.39: resolution: {integrity: sha512-BiEUfAiGCOllomsRAZOiMFP7LAnrifHpt56pc4Z7l9K6ACyN06Ns1JLMBxwkfLOjJRlSf06NwWsT7yzfpaVpyQ==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 '@types/qs': 6.9.9 '@types/range-parser': 1.2.6 '@types/send': 0.17.3 @@ -9540,7 +9546,7 @@ packages: /@types/graceful-fs@4.1.8: resolution: {integrity: sha512-NhRH7YzWq8WiNKVavKPBmtLYZHxNY19Hh+az28O/phfp68CF45pMFud+ZzJ8ewnxnC5smIdF3dqFeiSUQ5I+pw==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/hast@2.3.4: @@ -9700,7 +9706,7 @@ packages: /@types/node-fetch@2.6.7: resolution: {integrity: sha512-lX17GZVpJ/fuCjguZ5b3TjEbSENxmEk1B2z02yoXSK9WMEWRivhdSY73wWMn6bpcCDAOh6qAdktpKHIlkDk2lg==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 form-data: 4.0.0 dev: true @@ -9727,8 +9733,8 @@ packages: undici-types: 5.26.5 dev: true - /@types/node@20.11.27: - resolution: {integrity: sha512-qyUZfMnCg1KEz57r7pzFtSGt49f6RPkPBis3Vo4PbS7roQEDn22hiHzl/Lo1q4i4hDEgBJmBF/NTNg2XR0HbFg==} + /@types/node@20.11.30: + resolution: {integrity: sha512-dHM6ZxwlmuZaRmUPfv1p+KrdD1Dci04FbdEm/9wEMouFqxYoFl5aMkt0VMAUtYRQDyYvD41WJLukhq/ha3YuTw==} dependencies: undici-types: 5.26.5 dev: true @@ -9892,7 +9898,7 @@ packages: /@types/resolve@1.17.1: resolution: {integrity: sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw==} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/resolve@1.20.2: @@ -9910,7 +9916,7 @@ packages: resolution: {integrity: sha512-/7fKxvKUoETxjFUsuFlPB9YndePpxxRAOfGC/yJdc9kTjTeP5kRCTzfnE8kPUKCeyiyIZu0YQ76s50hCedI1ug==} dependencies: '@types/mime': 1.3.4 - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/serve-static@1.15.4: @@ -9918,7 +9924,7 @@ packages: dependencies: '@types/http-errors': 2.0.3 '@types/mime': 3.0.3 - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /@types/set-cookie-parser@2.4.2: @@ -18937,7 +18943,7 @@ packages: '@jest/environment': 27.5.1 '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 co: 4.6.0 dedent: 0.7.0 @@ -19072,7 +19078,7 @@ packages: '@jest/environment': 27.5.1 '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 jest-mock: 27.5.1 jest-util: 27.5.1 jsdom: 16.7.0 @@ -19090,7 +19096,7 @@ packages: '@jest/environment': 27.5.1 '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 jest-mock: 27.5.1 jest-util: 27.5.1 dev: true @@ -19134,7 +19140,7 @@ packages: dependencies: '@jest/types': 27.5.1 '@types/graceful-fs': 4.1.8 - '@types/node': 20.11.27 + '@types/node': 20.11.30 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -19174,7 +19180,7 @@ packages: '@jest/source-map': 27.5.1 '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 co: 4.6.0 expect: 27.5.1 @@ -19254,7 +19260,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 dev: true /jest-pnp-resolver@1.2.3(jest-resolve@27.5.1): @@ -19315,7 +19321,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 emittery: 0.8.1 graceful-fs: 4.2.11 @@ -19380,7 +19386,7 @@ packages: resolution: {integrity: sha512-jZCyo6iIxO1aqUxpuBlwTDMkzOAJS4a3eYz3YzgxxVQFwLeSA7Jfq5cbqCY+JLvTDrWirgusI/0KwxKMgrdf7w==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: - '@types/node': 20.11.27 + '@types/node': 20.11.30 graceful-fs: 4.2.11 dev: true @@ -19431,7 +19437,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -19468,7 +19474,7 @@ packages: dependencies: '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.11.27 + '@types/node': 20.11.30 ansi-escapes: 4.3.2 chalk: 4.1.2 jest-util: 27.5.1 @@ -27591,7 +27597,6 @@ packages: /workbox-google-analytics@7.0.0: resolution: {integrity: sha512-MEYM1JTn/qiC3DbpvP2BVhyIH+dV/5BjHk756u9VbwuAhu0QHyKscTnisQuz21lfRpOwiS9z4XdqeVAKol0bzg==} - deprecated: It is not compatible with newer versions of GA starting with v4, as long as you are using GAv3 it should be ok, but the package is not longer being maintained dependencies: workbox-background-sync: 7.0.0 workbox-core: 7.0.0 diff --git a/test/benchmark-wip/fixtures/basic/basic.bench.ts b/test/benchmark-wip/fixtures/basic/basic.bench.ts new file mode 100644 index 000000000000..b19590f142ef --- /dev/null +++ b/test/benchmark-wip/fixtures/basic/basic.bench.ts @@ -0,0 +1,13 @@ +import { bench, describe } from 'vitest' + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +describe('suite', () => { + bench('sleep(10)', async () => { + await sleep(10) + }, { time: 20, iterations: 0 }) + + bench('sleep(100)', async () => { + await sleep(100); + }, { time: 200, iterations: 0 }) +}) diff --git a/test/benchmark-wip/fixtures/basic/bench.json b/test/benchmark-wip/fixtures/basic/bench.json new file mode 100644 index 000000000000..d5b3f9c3a497 --- /dev/null +++ b/test/benchmark-wip/fixtures/basic/bench.json @@ -0,0 +1,44 @@ +{ + "-566580035_0_0": { + "name": "sleep(10)", + "rank": 1, + "rme": 0.8909326891329642, + "totalTime": 20.33896300000015, + "min": 10.162353000000167, + "max": 10.176609999999982, + "hz": 98.33343027370596, + "period": 10.169481500000074, + "mean": 10.169481500000074, + "variance": 0.00010163102449737513, + "sd": 0.010081221379246422, + "sem": 0.007128499999907944, + "df": 1, + "critical": 12.71, + "moe": 0.09060323499882997, + "p75": 10.176609999999982, + "p99": 10.176609999999982, + "p995": 10.176609999999982, + "p999": 10.176609999999982 + }, + "-566580035_0_1": { + "name": "sleep(100)", + "rank": 2, + "rme": 0.019683836914289064, + "totalTime": 200.49216100000012, + "min": 100.24452800000017, + "max": 100.24763299999995, + "hz": 9.975452356962718, + "period": 100.24608050000006, + "mean": 100.24608050000006, + "variance": 0.0000048205124993095935, + "sd": 0.0021955665554270025, + "sem": 0.0015524999998888232, + "df": 1, + "critical": 12.71, + "moe": 0.019732274998586943, + "p75": 100.24763299999995, + "p99": 100.24763299999995, + "p995": 100.24763299999995, + "p999": 100.24763299999995 + } +} \ No newline at end of file diff --git a/test/benchmark-wip/fixtures/basic/vitest.config.ts b/test/benchmark-wip/fixtures/basic/vitest.config.ts new file mode 100644 index 000000000000..abed6b2116e1 --- /dev/null +++ b/test/benchmark-wip/fixtures/basic/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({}) diff --git a/test/benchmark-wip/package.json b/test/benchmark-wip/package.json new file mode 100644 index 000000000000..347d05cabe30 --- /dev/null +++ b/test/benchmark-wip/package.json @@ -0,0 +1,11 @@ +{ + "name": "@vitest/benchmark-wip", + "type": "module", + "private": true, + "scripts": { + "test": "vitest" + }, + "devDependencies": { + "vitest": "workspace:*" + } +} diff --git a/test/benchmark-wip/test/compare.test.ts b/test/benchmark-wip/test/compare.test.ts new file mode 100644 index 000000000000..e3f158f23e3b --- /dev/null +++ b/test/benchmark-wip/test/compare.test.ts @@ -0,0 +1,49 @@ +import fs from 'node:fs' +import { expect, test } from 'vitest' +import { runVitest } from '../../test-utils' + +test('compare', { timeout: 60_000 }, async () => { + await fs.promises.rm('./fixtures/basic/bench.json', { force: true }) + + // force tty output + process.stdout.isTTY = true + + { + const result = await runVitest({ + root: './fixtures/basic', + outputFile: './bench.json', + reporters: ['default'], + }, [], 'benchmark') + expect(result.exitCode).toBe(0) + expect(fs.existsSync('./fixtures/basic/bench.json')).toBe(true) + } + + // --compare + { + const result = await runVitest({ + root: './fixtures/basic', + compare: './bench.json', + reporters: ['default'], + }, [], 'benchmark') + expect(result.exitCode).toBe(0) + + // assert a few lines after "✓ suite" in a following output + // + // ✓ basic.bench.ts (2) 830ms + // ✓ suite (2) 829ms + // name hz min max mean p75 p99 p995 p999 rme samples + // · sleep(10) 98.5536 10.1310 10.1625 10.1468 10.1625 10.1625 10.1625 10.1625 ±1.97% 2 [1.00x] ⇓ + // 98.6277 10.1212 10.1571 10.1391 10.1571 10.1571 10.1571 10.1571 ±2.25% - (baseline) + // · sleep(100) 9.9755 100.23 100.26 100.25 100.26 100.26 100.26 100.26 ±0.19% 2 [1.00x] ⇓ + // 9.9816 100.17 100.20 100.18 100.20 100.20 100.20 100.20 ±0.15% - (baseline) + + const lines = result.stdout.split('✓ suite')[1].split('\n').slice(1, 6) + expect(lines).toMatchObject([ + expect.stringMatching(/name/), + expect.stringMatching(/sleep.*(⇑|⇓)/), + expect.stringMatching(/baseline/), + expect.stringMatching(/sleep.*(⇑|⇓)/), + expect.stringMatching(/baseline/), + ]) + } +}) diff --git a/test/benchmark-wip/vitest.config.ts b/test/benchmark-wip/vitest.config.ts new file mode 100644 index 000000000000..199ecbae484f --- /dev/null +++ b/test/benchmark-wip/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + dir: './test', + env: { + // force tty output + CI: 'false', + }, + }, +}) From 5c7e9ca05491aeda225ce4616f06eefcd068c0b4 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 20 Mar 2024 10:21:52 +0900 Subject: [PATCH 15/33] test: skip ci --- .gitignore | 2 +- test/benchmark-wip/fixtures/basic/bench.json | 44 -------------------- test/benchmark-wip/test/compare.test.ts | 8 ++-- test/benchmark-wip/vitest.config.ts | 4 -- 4 files changed, 5 insertions(+), 53 deletions(-) delete mode 100644 test/benchmark-wip/fixtures/basic/bench.json diff --git a/.gitignore b/.gitignore index e6fecff012e2..e16136947717 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ dist ltex* .DS_Store bench/test/*/*/ -**/benchmark/bench.json +bench.json **/browser/browser.json cypress/videos cypress/downloads diff --git a/test/benchmark-wip/fixtures/basic/bench.json b/test/benchmark-wip/fixtures/basic/bench.json deleted file mode 100644 index d5b3f9c3a497..000000000000 --- a/test/benchmark-wip/fixtures/basic/bench.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "-566580035_0_0": { - "name": "sleep(10)", - "rank": 1, - "rme": 0.8909326891329642, - "totalTime": 20.33896300000015, - "min": 10.162353000000167, - "max": 10.176609999999982, - "hz": 98.33343027370596, - "period": 10.169481500000074, - "mean": 10.169481500000074, - "variance": 0.00010163102449737513, - "sd": 0.010081221379246422, - "sem": 0.007128499999907944, - "df": 1, - "critical": 12.71, - "moe": 0.09060323499882997, - "p75": 10.176609999999982, - "p99": 10.176609999999982, - "p995": 10.176609999999982, - "p999": 10.176609999999982 - }, - "-566580035_0_1": { - "name": "sleep(100)", - "rank": 2, - "rme": 0.019683836914289064, - "totalTime": 200.49216100000012, - "min": 100.24452800000017, - "max": 100.24763299999995, - "hz": 9.975452356962718, - "period": 100.24608050000006, - "mean": 100.24608050000006, - "variance": 0.0000048205124993095935, - "sd": 0.0021955665554270025, - "sem": 0.0015524999998888232, - "df": 1, - "critical": 12.71, - "moe": 0.019732274998586943, - "p75": 100.24763299999995, - "p99": 100.24763299999995, - "p995": 100.24763299999995, - "p999": 100.24763299999995 - } -} \ No newline at end of file diff --git a/test/benchmark-wip/test/compare.test.ts b/test/benchmark-wip/test/compare.test.ts index e3f158f23e3b..dfd3711a3db7 100644 --- a/test/benchmark-wip/test/compare.test.ts +++ b/test/benchmark-wip/test/compare.test.ts @@ -5,9 +5,7 @@ import { runVitest } from '../../test-utils' test('compare', { timeout: 60_000 }, async () => { await fs.promises.rm('./fixtures/basic/bench.json', { force: true }) - // force tty output - process.stdout.isTTY = true - + // --outputFile { const result = await runVitest({ root: './fixtures/basic', @@ -19,7 +17,9 @@ test('compare', { timeout: 60_000 }, async () => { } // --compare - { + // cannot run on CI since no tty output + if (!process.env.CI) { + process.stdout.isTTY = true const result = await runVitest({ root: './fixtures/basic', compare: './bench.json', diff --git a/test/benchmark-wip/vitest.config.ts b/test/benchmark-wip/vitest.config.ts index 199ecbae484f..fb6d42ab0931 100644 --- a/test/benchmark-wip/vitest.config.ts +++ b/test/benchmark-wip/vitest.config.ts @@ -3,9 +3,5 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ test: { dir: './test', - env: { - // force tty output - CI: 'false', - }, }, }) From 77d9736c8a042ef5c70fcaacad0f3e50bbd7b894 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 4 Apr 2024 17:56:43 +0900 Subject: [PATCH 16/33] chore: rename --- .gitignore | 1 - test/benchmark-wip/package.json | 11 ----------- test/benchmark-wip/vitest.config.ts | 7 ------- .../fixtures/compare}/basic.bench.ts | 0 .../fixtures/compare}/vitest.config.ts | 0 .../{benchmark-wip => benchmark}/test/compare.test.ts | 8 ++++---- 6 files changed, 4 insertions(+), 23 deletions(-) delete mode 100644 test/benchmark-wip/package.json delete mode 100644 test/benchmark-wip/vitest.config.ts rename test/{benchmark-wip/fixtures/basic => benchmark/fixtures/compare}/basic.bench.ts (100%) rename test/{benchmark-wip/fixtures/basic => benchmark/fixtures/compare}/vitest.config.ts (100%) rename test/{benchmark-wip => benchmark}/test/compare.test.ts (89%) diff --git a/.gitignore b/.gitignore index 03bfcc430754..67e90a1a3058 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ dist ltex* .DS_Store bench/test/*/*/ -bench.json **/bench.json **/browser/browser.json cypress/videos diff --git a/test/benchmark-wip/package.json b/test/benchmark-wip/package.json deleted file mode 100644 index 347d05cabe30..000000000000 --- a/test/benchmark-wip/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "@vitest/benchmark-wip", - "type": "module", - "private": true, - "scripts": { - "test": "vitest" - }, - "devDependencies": { - "vitest": "workspace:*" - } -} diff --git a/test/benchmark-wip/vitest.config.ts b/test/benchmark-wip/vitest.config.ts deleted file mode 100644 index fb6d42ab0931..000000000000 --- a/test/benchmark-wip/vitest.config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - dir: './test', - }, -}) diff --git a/test/benchmark-wip/fixtures/basic/basic.bench.ts b/test/benchmark/fixtures/compare/basic.bench.ts similarity index 100% rename from test/benchmark-wip/fixtures/basic/basic.bench.ts rename to test/benchmark/fixtures/compare/basic.bench.ts diff --git a/test/benchmark-wip/fixtures/basic/vitest.config.ts b/test/benchmark/fixtures/compare/vitest.config.ts similarity index 100% rename from test/benchmark-wip/fixtures/basic/vitest.config.ts rename to test/benchmark/fixtures/compare/vitest.config.ts diff --git a/test/benchmark-wip/test/compare.test.ts b/test/benchmark/test/compare.test.ts similarity index 89% rename from test/benchmark-wip/test/compare.test.ts rename to test/benchmark/test/compare.test.ts index dfd3711a3db7..ff327a660bbb 100644 --- a/test/benchmark-wip/test/compare.test.ts +++ b/test/benchmark/test/compare.test.ts @@ -3,17 +3,17 @@ import { expect, test } from 'vitest' import { runVitest } from '../../test-utils' test('compare', { timeout: 60_000 }, async () => { - await fs.promises.rm('./fixtures/basic/bench.json', { force: true }) + await fs.promises.rm('./fixtures/compare/bench.json', { force: true }) // --outputFile { const result = await runVitest({ - root: './fixtures/basic', + root: './fixtures/compare', outputFile: './bench.json', reporters: ['default'], }, [], 'benchmark') expect(result.exitCode).toBe(0) - expect(fs.existsSync('./fixtures/basic/bench.json')).toBe(true) + expect(fs.existsSync('./fixtures/compare/bench.json')).toBe(true) } // --compare @@ -21,7 +21,7 @@ test('compare', { timeout: 60_000 }, async () => { if (!process.env.CI) { process.stdout.isTTY = true const result = await runVitest({ - root: './fixtures/basic', + root: './fixtures/compare', compare: './bench.json', reporters: ['default'], }, [], 'benchmark') From 525a2a8a705fd699a57ecb2bc3e03c96d2f93968 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Thu, 4 Apr 2024 17:58:20 +0900 Subject: [PATCH 17/33] chore: lockfile --- pnpm-lock.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f742c37a9a85..64eb1502e0d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1541,12 +1541,6 @@ importers: specifier: workspace:* version: link:../../packages/vitest - test/benchmark-wip: - devDependencies: - vitest: - specifier: workspace:* - version: link:../../packages/vitest - test/browser: devDependencies: '@vitejs/plugin-basic-ssl': From 7377105b4182e1beb85b552b4eaefeff66a02fe7 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 9 Apr 2024 18:10:17 +0900 Subject: [PATCH 18/33] feat: compare on non-tty --- .../node/reporters/benchmark/table/index.ts | 28 ++++++++-------- .../benchmark/fixtures/compare/basic.bench.ts | 4 +-- test/benchmark/test/compare.test.ts | 33 +++++++------------ 3 files changed, 27 insertions(+), 38 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/index.ts b/packages/vitest/src/node/reporters/benchmark/table/index.ts index 1734c983049e..b353bf94f9e7 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/index.ts @@ -25,21 +25,21 @@ export class TableReporter extends BaseReporter { } async onCollected() { - if (this.isTTY) { - this.rendererOptions.logger = this.ctx.logger - this.rendererOptions.showHeap = this.ctx.config.logHeapUsage - this.rendererOptions.slowTestThreshold = this.ctx.config.slowTestThreshold - if (this.ctx.config.benchmark?.compare) { - const compareFile = pathe.resolve(this.ctx.config.root, this.ctx.config.benchmark?.compare) - try { - this.rendererOptions.compare = JSON.parse( - await fs.promises.readFile(compareFile, 'utf-8'), - ) - } - catch (e) { - this.ctx.logger.error(`Failed to read '${compareFile}'`, e) - } + this.rendererOptions.logger = this.ctx.logger + this.rendererOptions.showHeap = this.ctx.config.logHeapUsage + this.rendererOptions.slowTestThreshold = this.ctx.config.slowTestThreshold + if (this.ctx.config.benchmark?.compare) { + const compareFile = pathe.resolve(this.ctx.config.root, this.ctx.config.benchmark?.compare) + try { + this.rendererOptions.compare = JSON.parse( + await fs.promises.readFile(compareFile, 'utf-8'), + ) } + catch (e) { + this.ctx.logger.error(`Failed to read '${compareFile}'`, e) + } + } + if (this.isTTY) { const files = this.ctx.state.getFiles(this.watchFilters) if (!this.renderer) this.renderer = createTableRenderer(files, this.rendererOptions).start() diff --git a/test/benchmark/fixtures/compare/basic.bench.ts b/test/benchmark/fixtures/compare/basic.bench.ts index b19590f142ef..0fa89fef4ce1 100644 --- a/test/benchmark/fixtures/compare/basic.bench.ts +++ b/test/benchmark/fixtures/compare/basic.bench.ts @@ -3,11 +3,11 @@ import { bench, describe } from 'vitest' const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); describe('suite', () => { - bench('sleep(10)', async () => { + bench('sleep10', async () => { await sleep(10) }, { time: 20, iterations: 0 }) - bench('sleep(100)', async () => { + bench('sleep100', async () => { await sleep(100); }, { time: 200, iterations: 0 }) }) diff --git a/test/benchmark/test/compare.test.ts b/test/benchmark/test/compare.test.ts index ff327a660bbb..82288f247a4d 100644 --- a/test/benchmark/test/compare.test.ts +++ b/test/benchmark/test/compare.test.ts @@ -17,33 +17,22 @@ test('compare', { timeout: 60_000 }, async () => { } // --compare - // cannot run on CI since no tty output - if (!process.env.CI) { - process.stdout.isTTY = true + { const result = await runVitest({ root: './fixtures/compare', compare: './bench.json', reporters: ['default'], }, [], 'benchmark') expect(result.exitCode).toBe(0) - - // assert a few lines after "✓ suite" in a following output - // - // ✓ basic.bench.ts (2) 830ms - // ✓ suite (2) 829ms - // name hz min max mean p75 p99 p995 p999 rme samples - // · sleep(10) 98.5536 10.1310 10.1625 10.1468 10.1625 10.1625 10.1625 10.1625 ±1.97% 2 [1.00x] ⇓ - // 98.6277 10.1212 10.1571 10.1391 10.1571 10.1571 10.1571 10.1571 ±2.25% - (baseline) - // · sleep(100) 9.9755 100.23 100.26 100.25 100.26 100.26 100.26 100.26 ±0.19% 2 [1.00x] ⇓ - // 9.9816 100.17 100.20 100.18 100.20 100.20 100.20 100.20 ±0.15% - (baseline) - - const lines = result.stdout.split('✓ suite')[1].split('\n').slice(1, 6) - expect(lines).toMatchObject([ - expect.stringMatching(/name/), - expect.stringMatching(/sleep.*(⇑|⇓)/), - expect.stringMatching(/baseline/), - expect.stringMatching(/sleep.*(⇑|⇓)/), - expect.stringMatching(/baseline/), - ]) + const lines = result.stdout.split('\n').slice(3).slice(0, 6) + const expected = ` +✓ basic.bench.ts > suite + name + · sleep10 + (baseline) + · sleep100 + (baseline) + ` + expect(lines).toMatchObject(expected.trim().split('\n').map(s => expect.stringContaining(s.trim()))) } }) From e9f52f92fcfe2be7d02b0997c219efa811bf525e Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 9 Apr 2024 19:29:01 +0900 Subject: [PATCH 19/33] fix: bench only --compare option --- packages/vitest/src/node/cli/cac.ts | 31 +++++++++++++--------- packages/vitest/src/node/cli/cli-config.ts | 14 ++++++---- test/core/test/cli-test.test.ts | 20 +++++++++++++- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/packages/vitest/src/node/cli/cac.ts b/packages/vitest/src/node/cli/cac.ts index 8fd4587380aa..9744ae198b0e 100644 --- a/packages/vitest/src/node/cli/cac.ts +++ b/packages/vitest/src/node/cli/cac.ts @@ -1,5 +1,5 @@ import { normalize } from 'pathe' -import cac, { type CAC } from 'cac' +import cac, { type CAC, type Command } from 'cac' import c from 'picocolors' import { version } from '../../../package.json' import { toArray } from '../../utils' @@ -7,10 +7,10 @@ import type { Vitest, VitestRunMode } from '../../types' import { divider } from '../reporters/renderers/utils' import type { CliOptions } from './cli-api' import { startVitest } from './cli-api' -import type { CLIOption } from './cli-config' -import { cliOptionsConfig } from './cli-config' +import type { CLIOption, CLIOptions as CLIOptionsConfig } from './cli-config' +import { benchCliOptionsConfig, cliOptionsConfig } from './cli-config' -function addCommand(cli: CAC, name: string, option: CLIOption) { +function addCommand(cli: CAC | Command, name: string, option: CLIOption) { const commandName = option.alias || name let command = option.shorthand ? `-${option.shorthand}, --${commandName}` : `--${commandName}` if ('argument' in option) @@ -58,17 +58,21 @@ interface CLIOptions { allowUnknownOptions?: boolean } +function addCliOptions(cli: CAC | Command, options: CLIOptionsConfig) { + for (const optionName in options) { + const option = (options as any)[optionName] as CLIOption | null + if (option) + addCommand(cli, optionName, option) + } +} + export function createCLI(options: CLIOptions = {}) { const cli = cac('vitest') cli .version(version) - for (const optionName in cliOptionsConfig) { - const option = (cliOptionsConfig as any)[optionName] as CLIOption | null - if (option) - addCommand(cli, optionName, option) - } + addCliOptions(cli, cliOptionsConfig) cli.help((info) => { const helpSection = info.find(current => current.title?.startsWith('For more info, run any command')) @@ -160,9 +164,12 @@ export function createCLI(options: CLIOptions = {}) { .command('dev [...filters]', undefined, options) .action(watch) - cli - .command('bench [...filters]', undefined, options) - .action(benchmark) + addCliOptions( + cli + .command('bench [...filters]', undefined, options) + .action(benchmark), + benchCliOptionsConfig, + ) // TODO: remove in Vitest 2.0 cli diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index d3ce5517176f..69cc48cc50f5 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -21,7 +21,7 @@ export type CLIOption = { // require argument for non-boolean options (NonNullable extends boolean ? {} : { argument: string }) -type CLIOptions = { +export type CLIOptions = { [Key in keyof Config as NonNullable extends Function ? never : Key]-?: CLIOption | null } @@ -601,10 +601,6 @@ export const cliOptionsConfig: VitestCLIOptions = { clearScreen: { description: 'Clear terminal screen when re-running tests during watch mode (default: true)', }, - compare: { - description: 'benchmark output file to compare against', - argument: '', - }, // disable CLI options cliExclude: null, @@ -638,4 +634,12 @@ export const cliOptionsConfig: VitestCLIOptions = { deps: null, name: null, includeTaskLocation: null, + compare: null, +} + +export const benchCliOptionsConfig: Pick = { + compare: { + description: 'benchmark output file to compare against', + argument: '', + }, } diff --git a/test/core/test/cli-test.test.ts b/test/core/test/cli-test.test.ts index 1d4a3ce636cf..2ba389cfdd78 100644 --- a/test/core/test/cli-test.test.ts +++ b/test/core/test/cli-test.test.ts @@ -16,7 +16,7 @@ function parseArguments(commands: string, full = false) { delete options.color } - return { options, args } + return { options, args, matchedCommand: vitestCli.matchedCommand } } function getCLIOptions(commands: string) { @@ -131,6 +131,24 @@ test('fails when an array is passed down for a single value', async () => { .toThrowErrorMatchingInlineSnapshot(`[Error: Expected a single value for option "--coverage.provider ", received ["v8", "istanbul"]]`) }) +test('bench only options', async () => { + expect(() => + parseArguments('--compare file.json').matchedCommand?.checkUnknownOptions(), + ).toThrowErrorMatchingInlineSnapshot( + `[CACError: Unknown option \`--compare\`]`, + ) + + expect(() => + parseArguments( + 'bench --compare file.json', + ).matchedCommand?.checkUnknownOptions(), + ).not.toThrow() + + expect(parseArguments('bench --compare file.json').options).toEqual({ + compare: 'file.json', + }) +}) + test('even if coverage is boolean, don\'t fail', () => { expect(getCLIOptions('--coverage --coverage.provider v8').coverage).toEqual({ enabled: true, From 4f7f7079f74a22efc1732ba879bdcb498eaf3243 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Tue, 9 Apr 2024 19:32:03 +0900 Subject: [PATCH 20/33] refactor: minor --- packages/vitest/src/node/cli/cac.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/vitest/src/node/cli/cac.ts b/packages/vitest/src/node/cli/cac.ts index 9744ae198b0e..b413884bf167 100644 --- a/packages/vitest/src/node/cli/cac.ts +++ b/packages/vitest/src/node/cli/cac.ts @@ -59,8 +59,7 @@ interface CLIOptions { } function addCliOptions(cli: CAC | Command, options: CLIOptionsConfig) { - for (const optionName in options) { - const option = (options as any)[optionName] as CLIOption | null + for (const [optionName, option] of Object.entries(options)) { if (option) addCommand(cli, optionName, option) } From 86088a86076f0ea52df64bbf74018434e484ba2c Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 12 Apr 2024 10:52:49 +0900 Subject: [PATCH 21/33] refactor: add FormattedBenchamrkReport --- .../node/reporters/benchmark/table/index.ts | 60 +++++++++++++++---- .../reporters/benchmark/table/tableRender.ts | 12 ++-- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/index.ts b/packages/vitest/src/node/reporters/benchmark/table/index.ts index b353bf94f9e7..6d23bb095280 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/index.ts @@ -31,8 +31,10 @@ export class TableReporter extends BaseReporter { if (this.ctx.config.benchmark?.compare) { const compareFile = pathe.resolve(this.ctx.config.root, this.ctx.config.benchmark?.compare) try { - this.rendererOptions.compare = JSON.parse( - await fs.promises.readFile(compareFile, 'utf-8'), + this.rendererOptions.compare = flattenFormattedBenchamrkReport( + JSON.parse( + await fs.promises.readFile(compareFile, 'utf-8'), + ), ) } catch (e) { @@ -79,7 +81,7 @@ export class TableReporter extends BaseReporter { const outputDirectory = pathe.dirname(outputFile) if (!fs.existsSync(outputDirectory)) await fs.promises.mkdir(outputDirectory, { recursive: true }) - const output = createBenchmarkOutput(files) + const output = createFormattedBenchamrkReport(files) await fs.promises.writeFile(outputFile, JSON.stringify(output, null, 2)) } } @@ -107,18 +109,52 @@ export class TableReporter extends BaseReporter { } } -export interface TableBenchmarkOutput { - [id: string]: Omit +export interface FormattedBenchamrkReport { + groups: FormattedBenchmarkGroup[] } -function createBenchmarkOutput(files: File[]) { - const result: TableBenchmarkOutput = {} - for (const test of getTasks(files)) { - if (test.meta?.benchmark && test.result?.benchmark) { - // strip gigantic "samples" - const { samples: _samples, ...rest } = test.result.benchmark - result[test.id] = rest +export interface FlatBenchmarkReport { + [id: string]: FormattedBenchmarkResult +} + +interface FormattedBenchmarkGroup { + fullName: string + benchmarks: FormattedBenchmarkResult[] +} + +export type FormattedBenchmarkResult = Omit & { + id: string + sampleCount: number +} + +function createFormattedBenchamrkReport(files: File[]) { + const result: FormattedBenchamrkReport = { groups: [] } + for (const task of getTasks(files)) { + if (task && task.type === 'suite') { + const benches = task.tasks.filter(t => t.meta.benchmark && t.result?.benchmark) + if (benches.length > 0) { + result.groups.push({ + fullName: getFullName(task, ' > '), + benchmarks: benches.map((t) => { + const { samples, ...rest } = t.result!.benchmark! + return { + id: t.id, + sampleCount: samples.length, + ...rest, + } + }), + }) + } } } return result } + +function flattenFormattedBenchamrkReport(report: FormattedBenchamrkReport): FlatBenchmarkReport { + const flat: FlatBenchmarkReport = {} + for (const group of report.groups) { + for (const t of group.benchmarks) + flat[t.id] = t + } + return flat +} diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index 37cdc70ccb94..8b67af661a5d 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -6,14 +6,14 @@ import { getTests, notNullish } from '../../../../utils' import { F_RIGHT } from '../../../../utils/figures' import type { Logger } from '../../../logger' import { getCols, getStateSymbol } from '../../renderers/utils' -import type { TableBenchmarkOutput } from '.' +import type { FlatBenchmarkReport } from '.' export interface TableRendererOptions { renderSucceed?: boolean logger: Logger showHeap: boolean slowTestThreshold: number - compare?: TableBenchmarkOutput + compare?: FlatBenchmarkReport } const outputMap = new WeakMap() @@ -103,11 +103,11 @@ export function renderTree(tasks: Task[], options: TableRendererOptions, level = benchMap[t.id] = { current: t.result.benchmark, } - if (options.compare && options.compare[t.id]) { + const baseline = options.compare?.[t.id] + if (baseline) { benchMap[t.id].baseline = { - ...options.compare[t.id], - samples: [], - name: '', + ...baseline, + samples: Array(baseline.sampleCount), } } } From 8296ed74e4184a16c3c5a0cd966189ffe7713592 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 12 Apr 2024 18:24:37 +0900 Subject: [PATCH 22/33] feat: add filepath --- .../node/reporters/benchmark/table/index.ts | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/index.ts b/packages/vitest/src/node/reporters/benchmark/table/index.ts index 6d23bb095280..dcceaf510f7f 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/index.ts @@ -118,6 +118,7 @@ export interface FlatBenchmarkReport { } interface FormattedBenchmarkGroup { + filepath: string fullName: string benchmarks: FormattedBenchmarkResult[] } @@ -129,21 +130,28 @@ export type FormattedBenchmarkResult = Omit & { function createFormattedBenchamrkReport(files: File[]) { const result: FormattedBenchamrkReport = { groups: [] } - for (const task of getTasks(files)) { - if (task && task.type === 'suite') { - const benches = task.tasks.filter(t => t.meta.benchmark && t.result?.benchmark) - if (benches.length > 0) { - result.groups.push({ - fullName: getFullName(task, ' > '), - benchmarks: benches.map((t) => { - const { samples, ...rest } = t.result!.benchmark! - return { + for (const file of files) { + for (const task of getTasks(file)) { + if (task && task.type === 'suite') { + const benchmarks: FormattedBenchmarkResult[] = [] + for (const t of task.tasks) { + const benchmark = t.meta.benchmark && t.result?.benchmark + if (benchmark) { + const { samples, ...rest } = benchmark + benchmarks.push({ id: t.id, sampleCount: samples.length, ...rest, - } - }), - }) + }) + } + } + if (benchmarks.length) { + result.groups.push({ + filepath: file.filepath, + fullName: getFullName(task, ' > '), + benchmarks, + }) + } } } } From 3b329d4cd2a84f8c8d05b699d6b4dc5e374926ce Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 12 Apr 2024 18:34:58 +0900 Subject: [PATCH 23/33] chore: tweak format --- .../node/reporters/benchmark/table/index.ts | 27 ++++++++++++------- test/benchmark/package.json | 2 +- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/index.ts b/packages/vitest/src/node/reporters/benchmark/table/index.ts index dcceaf510f7f..c4aa15722918 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/index.ts @@ -110,15 +110,18 @@ export class TableReporter extends BaseReporter { } export interface FormattedBenchamrkReport { - groups: FormattedBenchmarkGroup[] + files: { + filepath: string + groups: FormattedBenchmarkGroup[] + }[] } +// flat results with TaskId as a key export interface FlatBenchmarkReport { [id: string]: FormattedBenchmarkResult } interface FormattedBenchmarkGroup { - filepath: string fullName: string benchmarks: FormattedBenchmarkResult[] } @@ -129,8 +132,9 @@ export type FormattedBenchmarkResult = Omit & { } function createFormattedBenchamrkReport(files: File[]) { - const result: FormattedBenchamrkReport = { groups: [] } + const report: FormattedBenchamrkReport = { files: [] } for (const file of files) { + const groups: FormattedBenchmarkGroup[] = [] for (const task of getTasks(file)) { if (task && task.type === 'suite') { const benchmarks: FormattedBenchmarkResult[] = [] @@ -146,23 +150,28 @@ function createFormattedBenchamrkReport(files: File[]) { } } if (benchmarks.length) { - result.groups.push({ - filepath: file.filepath, + groups.push({ fullName: getFullName(task, ' > '), benchmarks, }) } } } + report.files.push({ + filepath: file.filepath, + groups, + }) } - return result + return report } function flattenFormattedBenchamrkReport(report: FormattedBenchamrkReport): FlatBenchmarkReport { const flat: FlatBenchmarkReport = {} - for (const group of report.groups) { - for (const t of group.benchmarks) - flat[t.id] = t + for (const file of report.files) { + for (const group of file.groups) { + for (const t of group.benchmarks) + flat[t.id] = t + } } return flat } diff --git a/test/benchmark/package.json b/test/benchmark/package.json index 5566080b1da6..b1fee30693c5 100644 --- a/test/benchmark/package.json +++ b/test/benchmark/package.json @@ -1,5 +1,5 @@ { - "name": "@vitest/benchmark-sequential", + "name": "@vitest/test-benchmark", "type": "module", "private": true, "scripts": { From 0ef0bc822e95e4b07e23fe4aee12f0f0acf4b2ed Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sat, 13 Apr 2024 16:34:54 +0900 Subject: [PATCH 24/33] refactor: tweak --- .../vitest/src/node/reporters/benchmark/table/tableRender.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index 8b67af661a5d..2a49071b456d 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -49,8 +49,7 @@ function renderBenchmarkItems(result: BenchmarkResult) { formatNumber(result.p995 || 0), formatNumber(result.p999 || 0), `±${(result.rme || 0).toFixed(2)}%`, - // TODO: persist only sampleCount? - result.samples.length ? result.samples.length.toString() : '-', + result.samples.length.toString(), ] } From 147baf9129b7930aaed2faf781e1cafefb25b0c1 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sat, 13 Apr 2024 16:35:43 +0900 Subject: [PATCH 25/33] chore: lockfile --- pnpm-lock.yaml | 66 +++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 90fd6184cccd..7c52d85048bf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6076,7 +6076,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 chalk: 4.1.2 jest-message-util: 27.5.1 jest-util: 27.5.1 @@ -6097,7 +6097,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.8.1 @@ -6134,7 +6134,7 @@ packages: dependencies: '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 jest-mock: 27.5.1 dev: true @@ -6151,7 +6151,7 @@ packages: dependencies: '@jest/types': 27.5.1 '@sinonjs/fake-timers': 8.1.0 - '@types/node': 20.12.6 + '@types/node': 20.12.7 jest-message-util: 27.5.1 jest-mock: 27.5.1 jest-util: 27.5.1 @@ -6180,7 +6180,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 chalk: 4.1.2 collect-v8-coverage: 1.0.2 exit: 0.1.2 @@ -6311,7 +6311,7 @@ packages: dependencies: '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 20.12.6 + '@types/node': 20.12.7 '@types/yargs': 16.0.7 chalk: 4.1.2 dev: true @@ -9696,7 +9696,7 @@ packages: resolution: {integrity: sha512-N7UDG0/xiPQa2D/XrVJXjkWbpqHCd2sBaB32ggRF2l83RhPfamgKGF8gwwqyksS95qUS5ZYF9aF+lLPRlwI2UA==} dependencies: '@types/connect': 3.4.37 - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true /@types/braces@3.0.1: @@ -9717,7 +9717,7 @@ packages: /@types/connect@3.4.37: resolution: {integrity: sha512-zBUSRqkfZ59OcwXon4HVxhx5oWCJmc0OtBTK05M+p0dYjgN6iTwIL2T/WbsQZrEsdnwaF9cWQ+azOnpPvIqY3Q==} dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true /@types/cookie@0.4.1: @@ -9784,7 +9784,7 @@ packages: /@types/express-serve-static-core@4.17.39: resolution: {integrity: sha512-BiEUfAiGCOllomsRAZOiMFP7LAnrifHpt56pc4Z7l9K6ACyN06Ns1JLMBxwkfLOjJRlSf06NwWsT7yzfpaVpyQ==} dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 '@types/qs': 6.9.9 '@types/range-parser': 1.2.6 '@types/send': 0.17.3 @@ -9808,7 +9808,7 @@ packages: requiresBuild: true dependencies: '@types/jsonfile': 6.1.4 - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true optional: true @@ -9841,13 +9841,13 @@ packages: /@types/graceful-fs@4.1.5: resolution: {integrity: sha512-anKkLmZZ+xm4p8JWBf4hElkM4XR+EZeA2M9BAkkTldmcyDY4mbdIJnRghDJH3Ov5ooY7/UAoENtmdMSkaAd7Cw==} dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true /@types/graceful-fs@4.1.8: resolution: {integrity: sha512-NhRH7YzWq8WiNKVavKPBmtLYZHxNY19Hh+az28O/phfp68CF45pMFud+ZzJ8ewnxnC5smIdF3dqFeiSUQ5I+pw==} dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true /@types/hast@2.3.4: @@ -9947,7 +9947,7 @@ packages: resolution: {integrity: sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==} requiresBuild: true dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true optional: true @@ -10018,7 +10018,7 @@ packages: /@types/node-fetch@2.6.7: resolution: {integrity: sha512-lX17GZVpJ/fuCjguZ5b3TjEbSENxmEk1B2z02yoXSK9WMEWRivhdSY73wWMn6bpcCDAOh6qAdktpKHIlkDk2lg==} dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 form-data: 4.0.0 dev: true @@ -10056,8 +10056,8 @@ packages: undici-types: 5.26.5 dev: true - /@types/node@20.12.6: - resolution: {integrity: sha512-3KurE8taB8GCvZBPngVbp0lk5CKi8M9f9k1rsADh0Evdz5SzJ+Q+Hx9uHoFGsLnLnd1xmkDQr2hVhlA0Mn0lKQ==} + /@types/node@20.12.7: + resolution: {integrity: sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==} dependencies: undici-types: 5.26.5 dev: true @@ -10212,7 +10212,7 @@ packages: /@types/resolve@1.17.1: resolution: {integrity: sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw==} dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true /@types/resolve@1.20.2: @@ -10230,7 +10230,7 @@ packages: resolution: {integrity: sha512-/7fKxvKUoETxjFUsuFlPB9YndePpxxRAOfGC/yJdc9kTjTeP5kRCTzfnE8kPUKCeyiyIZu0YQ76s50hCedI1ug==} dependencies: '@types/mime': 1.3.4 - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true /@types/serve-static@1.15.4: @@ -10238,7 +10238,7 @@ packages: dependencies: '@types/http-errors': 2.0.3 '@types/mime': 3.0.3 - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true /@types/set-cookie-parser@2.4.2: @@ -10393,7 +10393,7 @@ packages: resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} requiresBuild: true dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true optional: true @@ -19425,7 +19425,7 @@ packages: '@jest/environment': 27.5.1 '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 chalk: 4.1.2 co: 4.6.0 dedent: 0.7.0 @@ -19560,7 +19560,7 @@ packages: '@jest/environment': 27.5.1 '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 jest-mock: 27.5.1 jest-util: 27.5.1 jsdom: 16.7.0 @@ -19578,7 +19578,7 @@ packages: '@jest/environment': 27.5.1 '@jest/fake-timers': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 jest-mock: 27.5.1 jest-util: 27.5.1 dev: true @@ -19622,7 +19622,7 @@ packages: dependencies: '@jest/types': 27.5.1 '@types/graceful-fs': 4.1.8 - '@types/node': 20.12.6 + '@types/node': 20.12.7 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -19662,7 +19662,7 @@ packages: '@jest/source-map': 27.5.1 '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 chalk: 4.1.2 co: 4.6.0 expect: 27.5.1 @@ -19742,7 +19742,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true /jest-pnp-resolver@1.2.3(jest-resolve@27.5.1): @@ -19803,7 +19803,7 @@ packages: '@jest/test-result': 27.5.1 '@jest/transform': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 chalk: 4.1.2 emittery: 0.8.1 graceful-fs: 4.2.11 @@ -19860,7 +19860,7 @@ packages: resolution: {integrity: sha512-S5wqyz0DXnNJPd/xfIzZ5Xnp1HrJWBczg8mMfMpN78OJ5eDxXyf+Ygld9wX1DnUWbIbhM1YDY95NjR4CBXkb2g==} engines: {node: '>= 10.14.2'} dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 graceful-fs: 4.2.11 dev: true @@ -19868,7 +19868,7 @@ packages: resolution: {integrity: sha512-jZCyo6iIxO1aqUxpuBlwTDMkzOAJS4a3eYz3YzgxxVQFwLeSA7Jfq5cbqCY+JLvTDrWirgusI/0KwxKMgrdf7w==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 graceful-fs: 4.2.11 dev: true @@ -19919,7 +19919,7 @@ packages: engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} dependencies: '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -19956,7 +19956,7 @@ packages: dependencies: '@jest/test-result': 27.5.1 '@jest/types': 27.5.1 - '@types/node': 20.12.6 + '@types/node': 20.12.7 ansi-escapes: 4.3.2 chalk: 4.1.2 jest-util: 27.5.1 @@ -19967,7 +19967,7 @@ packages: resolution: {integrity: sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ==} engines: {node: '>= 10.13.0'} dependencies: - '@types/node': 20.12.6 + '@types/node': 20.12.7 merge-stream: 2.0.0 supports-color: 7.2.0 dev: true @@ -28840,7 +28840,7 @@ packages: hasBin: true optionalDependencies: '@types/fs-extra': 11.0.4 - '@types/node': 20.12.6 + '@types/node': 20.12.7 dev: true file:test/config/fixtures/conditions-subpackage: From 2186c0c3d86f6ea4675ba5c97c2b332eb2f2cc43 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sat, 13 Apr 2024 16:51:39 +0900 Subject: [PATCH 26/33] feat: add --outputJson --- packages/vitest/src/defaults.ts | 2 +- packages/vitest/src/node/cli/cli-config.ts | 7 ++++++- packages/vitest/src/node/config.ts | 2 ++ .../vitest/src/node/reporters/benchmark/table/index.ts | 3 +-- packages/vitest/src/types/benchmark.ts | 5 +++++ packages/vitest/src/types/config.ts | 7 ++++++- test/benchmark/test/compare.test.ts | 4 ++-- 7 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/vitest/src/defaults.ts b/packages/vitest/src/defaults.ts index 5c895403fe7a..1c94d2835165 100644 --- a/packages/vitest/src/defaults.ts +++ b/packages/vitest/src/defaults.ts @@ -4,7 +4,7 @@ import { isCI } from './utils/env' export const defaultInclude = ['**/*.{test,spec}.?(c|m)[jt]s?(x)'] export const defaultExclude = ['**/node_modules/**', '**/dist/**', '**/cypress/**', '**/.{idea,git,cache,output,temp}/**', '**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*'] -export const benchmarkConfigDefaults: Required> = { +export const benchmarkConfigDefaults: Required> = { include: ['**/*.{bench,benchmark}.?(c|m)[jt]s?(x)'], exclude: defaultExclude, includeSource: [], diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 69cc48cc50f5..a9d6e3be0832 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -635,11 +635,16 @@ export const cliOptionsConfig: VitestCLIOptions = { name: null, includeTaskLocation: null, compare: null, + outputJson: null, } -export const benchCliOptionsConfig: Pick = { +export const benchCliOptionsConfig: Pick = { compare: { description: 'benchmark output file to compare against', argument: '', }, + outputJson: { + description: 'benchmark output file', + argument: '', + }, } diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index b79c7403d300..ac70de06c167 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -377,6 +377,8 @@ export function resolveConfig( // --compare from cli if (options.compare) resolved.benchmark.compare = options.compare + if (options.outputJson) + resolved.benchmark.outputJson = options.outputJson } resolved.setupFiles = toArray(resolved.setupFiles || []).map(file => diff --git a/packages/vitest/src/node/reporters/benchmark/table/index.ts b/packages/vitest/src/node/reporters/benchmark/table/index.ts index c4aa15722918..e33faf577725 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/index.ts @@ -6,7 +6,6 @@ import type { UserConsoleLog } from '../../../../types/general' import { BaseReporter } from '../../base' import type { BenchmarkResult, File } from '../../../../types' import { getFullName, getTasks } from '../../../../utils' -import { getOutputFile } from '../../../../utils/config-helpers' import { getStateSymbol } from '../../renderers/utils' import { type TableRendererOptions, createTableRenderer, renderTree } from './tableRender' @@ -75,7 +74,7 @@ export class TableReporter extends BaseReporter { await super.onFinished(files, errors) // write output for future comparison - let outputFile = getOutputFile(this.ctx.config.benchmark, 'default') + let outputFile = this.ctx.config.benchmark?.outputJson if (outputFile) { outputFile = pathe.resolve(this.ctx.config.root, outputFile) const outputDirectory = pathe.dirname(outputFile) diff --git a/packages/vitest/src/types/benchmark.ts b/packages/vitest/src/types/benchmark.ts index df232dd05294..a985e15146d6 100644 --- a/packages/vitest/src/types/benchmark.ts +++ b/packages/vitest/src/types/benchmark.ts @@ -44,6 +44,11 @@ export interface BenchmarkUserOptions { * benchmark output file to compare against */ compare?: string + + /** + * benchmark output file + */ + outputJson?: string } export interface Benchmark extends Custom { diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index d81a6551d7b9..1733b5e98328 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -851,6 +851,11 @@ export interface UserConfig extends InlineConfig { * benchmark.compare option exposed at the top level for cli */ compare?: string + + /** + * benchmark.outputJson option exposed at the top level for cli + */ + outputJson?: string } export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'browser' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'benchmark' | 'shard' | 'cache' | 'sequence' | 'typecheck' | 'runner' | 'poolOptions' | 'pool' | 'cliExclude'> { @@ -877,7 +882,7 @@ export interface ResolvedConfig extends Omit, 'config' | 'f api?: ApiConfig cliExclude?: string[] - benchmark?: Required> & Pick + benchmark?: Required> & Pick shard?: { index: number count: number diff --git a/test/benchmark/test/compare.test.ts b/test/benchmark/test/compare.test.ts index 82288f247a4d..5e48a4065592 100644 --- a/test/benchmark/test/compare.test.ts +++ b/test/benchmark/test/compare.test.ts @@ -5,11 +5,11 @@ import { runVitest } from '../../test-utils' test('compare', { timeout: 60_000 }, async () => { await fs.promises.rm('./fixtures/compare/bench.json', { force: true }) - // --outputFile + // --outputJson { const result = await runVitest({ root: './fixtures/compare', - outputFile: './bench.json', + outputJson: './bench.json', reporters: ['default'], }, [], 'benchmark') expect(result.exitCode).toBe(0) From e3a060779e9a1e8b2591b0db2ee2bcc4b534335b Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sat, 13 Apr 2024 16:55:39 +0900 Subject: [PATCH 27/33] chore: log outputFile --- packages/vitest/src/node/reporters/benchmark/table/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/vitest/src/node/reporters/benchmark/table/index.ts b/packages/vitest/src/node/reporters/benchmark/table/index.ts index e33faf577725..12a06fca5ca8 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/index.ts @@ -82,6 +82,7 @@ export class TableReporter extends BaseReporter { await fs.promises.mkdir(outputDirectory, { recursive: true }) const output = createFormattedBenchamrkReport(files) await fs.promises.writeFile(outputFile, JSON.stringify(output, null, 2)) + this.ctx.logger.log(`Benchmark report written to ${outputFile}`) } } From c3ce69186b0535045ec7aaab21c0de08059ea162 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sat, 13 Apr 2024 17:08:43 +0900 Subject: [PATCH 28/33] test: update --- test/benchmark/test/basic.test.ts | 90 +++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/test/benchmark/test/basic.test.ts b/test/benchmark/test/basic.test.ts index 46021e7483f4..b4cac136a14e 100644 --- a/test/benchmark/test/basic.test.ts +++ b/test/benchmark/test/basic.test.ts @@ -1,6 +1,7 @@ import fs from 'node:fs' import { expect, it } from 'vitest' import * as pathe from 'pathe' +import type { FormattedBenchamrkReport } from 'vitest/src/node/reporters/benchmark/table/index.js' import { runVitest } from '../../test-utils' it('basic', { timeout: 60_000 }, async () => { @@ -8,31 +9,76 @@ it('basic', { timeout: 60_000 }, async () => { const benchFile = pathe.join(root, 'bench.json') fs.rmSync(benchFile, { force: true }) - await runVitest({ + const result = await runVitest({ root, allowOnly: true, - benchmark: { - reporters: 'json', - outputFile: 'bench.json', - }, + outputJson: 'bench.json', }, [], 'benchmark') + expect(result.exitCode).toBe(0) const benchResult = await fs.promises.readFile(benchFile, 'utf-8') - const resultJson = JSON.parse(benchResult) - - expect(Object.keys(resultJson.testResults)).toEqual( - expect.arrayContaining([ - 'sort', - 'timeout', - 'a0', - 'c1', - 'a2', - 'b3', - 'b4', - ]), - ) - - const skipped = ['skip', 's0', 's1', 's2', 's3', 'sb4', 's4', 'unimplemented suite', 'unimplemented test'] - for (const b of skipped) - expect(benchResult).not.toContain(b) + const resultJson: FormattedBenchamrkReport = JSON.parse(benchResult) + const names = resultJson.files.map(f => f.groups.map(g => [g.fullName, g.benchmarks.map(b => b.name)])) + expect(names).toMatchInlineSnapshot(` + [ + [ + [ + "base.bench.ts > sort", + [ + "normal", + "reverse", + ], + ], + [ + "base.bench.ts > timeout", + [ + "timeout100", + "timeout75", + "timeout50", + "timeout25", + ], + ], + ], + [], + [ + [ + "only.bench.ts", + [ + "visited", + "visited2", + ], + ], + [ + "only.bench.ts > a0", + [ + "0", + ], + ], + [ + "only.bench.ts > a1 > b1 > c1", + [ + "1", + ], + ], + [ + "only.bench.ts > a2", + [ + "2", + ], + ], + [ + "only.bench.ts > a3 > b3", + [ + "3", + ], + ], + [ + "only.bench.ts > a4 > b4", + [ + "4", + ], + ], + ], + ] + `) }) From 35931edc1f3ea694e3396ff822975257a99c9ba2 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sat, 13 Apr 2024 17:13:30 +0900 Subject: [PATCH 29/33] chore: remove json reporter --- .../src/node/reporters/benchmark/index.ts | 2 - .../src/node/reporters/benchmark/json.ts | 82 ------------------- 2 files changed, 84 deletions(-) delete mode 100644 packages/vitest/src/node/reporters/benchmark/json.ts diff --git a/packages/vitest/src/node/reporters/benchmark/index.ts b/packages/vitest/src/node/reporters/benchmark/index.ts index 00aa184557b4..554c5d1998cf 100644 --- a/packages/vitest/src/node/reporters/benchmark/index.ts +++ b/packages/vitest/src/node/reporters/benchmark/index.ts @@ -1,10 +1,8 @@ import { VerboseReporter } from '../verbose' -import { JsonReporter } from './json' import { TableReporter } from './table' export const BenchmarkReportsMap = { default: TableReporter, verbose: VerboseReporter, - json: JsonReporter, } export type BenchmarkBuiltinReporters = keyof typeof BenchmarkReportsMap diff --git a/packages/vitest/src/node/reporters/benchmark/json.ts b/packages/vitest/src/node/reporters/benchmark/json.ts deleted file mode 100644 index 23bcb87d010f..000000000000 --- a/packages/vitest/src/node/reporters/benchmark/json.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { existsSync, promises as fs } from 'node:fs' -import { dirname, resolve } from 'pathe' -import type { Vitest } from '../../../node' -import type { BenchTaskResult, File, Reporter } from '../../../types' -import { getSuites, getTests } from '../../../utils' -import { getOutputFile } from '../../../utils/config-helpers' - -interface FormattedTestResults { - numTotalTests: number - numTotalTestSuites: number - testResults: Record -} - -export class JsonReporter implements Reporter { - start = 0 - ctx!: Vitest - - onInit(ctx: Vitest): void { - this.ctx = ctx - } - - protected async logTasks(files: File[]) { - const suites = getSuites(files) - const numTotalTestSuites = suites.length - const tests = getTests(files) - const numTotalTests = tests.length - const testResults: Record = {} - const outputFile = getOutputFile(this.ctx.config.benchmark, 'json') - for (const file of files) { - const tests = getTests([file]) - for (const test of tests) { - const res = test.result?.benchmark - if (!res || test.mode === 'skip') // TODO mark as skipped - continue - if (!outputFile) - res.samples = 'ignore on terminal' as any - testResults[test.suite!.name] = (testResults[test.suite!.name] || []).concat(res) - } - - if (tests.some(t => t.result?.state === 'run')) { - this.ctx.logger.warn('WARNING: Some tests are still running when generating the json report.' - + 'This is likely an internal bug in Vitest.' - + 'Please report it to https://github.com/vitest-dev/vitest/issues') - } - } - - const result: FormattedTestResults = { - numTotalTestSuites, - numTotalTests, - testResults, - } - - await this.writeReport(JSON.stringify(result, null, 2)) - } - - async onFinished(files = this.ctx.state.getFiles()) { - await this.logTasks(files) - } - - /** - * Writes the report to an output file if specified in the config, - * or logs it to the console otherwise. - * @param report - */ - async writeReport(report: string) { - const outputFile = getOutputFile(this.ctx.config.benchmark, 'json') - - if (outputFile) { - const reportFile = resolve(this.ctx.config.root, outputFile) - - const outputDirectory = dirname(reportFile) - if (!existsSync(outputDirectory)) - await fs.mkdir(outputDirectory, { recursive: true }) - - await fs.writeFile(reportFile, report, 'utf-8') - this.ctx.logger.log(`json report written to ${reportFile}`) - } - else { - this.ctx.logger.log(report) - } - } -} From b03080ac16efb507717feaf55a338115fae47b75 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sat, 13 Apr 2024 17:30:33 +0900 Subject: [PATCH 30/33] docs: options --- docs/config/index.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/config/index.md b/docs/config/index.md index 7d1f011d2541..a34ae74a3330 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -312,6 +312,20 @@ By providing an object instead of a string you can define individual outputs whe To provide object via CLI command, use the following syntax: `--outputFile.json=./path --outputFile.junit=./other-path`. +#### benchmark.outputJson + +- **Type:** `string | undefined` +- **Default:** `undefined` + +A file path to store the benchmark result, which can be used for `--compare` option later. + +#### benchmark.compare + +- **Type:** `string | undefined` +- **Default:** `undefined` + +A file path to a previous benchmark result to compare against current runs. + ### alias - **Type:** `Record | Array<{ find: string | RegExp, replacement: string, customResolver?: ResolverFunction | ResolverObject }>` From 5fbfb6c99dabad7d06e641844e5a2f7cec2942ea Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sat, 13 Apr 2024 17:40:27 +0900 Subject: [PATCH 31/33] docs: benchmark report screenshots --- docs/guide/features.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/guide/features.md b/docs/guide/features.md index 39c1bee5abdd..db0be1c40cbe 100644 --- a/docs/guide/features.md +++ b/docs/guide/features.md @@ -210,6 +210,9 @@ describe('sort', () => { }) ``` +Benchmark report +Benchmark report + ## Type Testing Experimental {#type-testing} Since Vitest 0.25.0 you can [write tests](/guide/testing-types) to catch type regressions. Vitest comes with [`expect-type`](https://github.com/mmkal/expect-type) package to provide you with a similar and easy to understand API. From 34e90f303a9fbfca26398ad17cd1899e21d597f2 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sat, 13 Apr 2024 17:46:47 +0900 Subject: [PATCH 32/33] docs: examples --- docs/config/index.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/config/index.md b/docs/config/index.md index a34ae74a3330..a16a53c83968 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -319,6 +319,18 @@ To provide object via CLI command, use the following syntax: `--outputFile.json= A file path to store the benchmark result, which can be used for `--compare` option later. +For example: + +```sh +# save main branch's result +git checkout main +vitest bench --outputJson main.json + +# change a branch and compare against main +git checkout feature +vitest bench --compare main.json +``` + #### benchmark.compare - **Type:** `string | undefined` From 5d4a80623da7064cefee2d2fc80e501d044901b0 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Thu, 2 May 2024 15:13:18 +0200 Subject: [PATCH 33/33] Apply suggestions from code review --- docs/config/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config/index.md b/docs/config/index.md index a16a53c83968..584a590804e6 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -312,7 +312,7 @@ By providing an object instead of a string you can define individual outputs whe To provide object via CLI command, use the following syntax: `--outputFile.json=./path --outputFile.junit=./other-path`. -#### benchmark.outputJson +#### benchmark.outputJson 1.6.0 {#benchmark-outputJson} - **Type:** `string | undefined` - **Default:** `undefined` @@ -331,7 +331,7 @@ git checkout feature vitest bench --compare main.json ``` -#### benchmark.compare +#### benchmark.compare 1.6.0 {#benchmark-compare} - **Type:** `string | undefined` - **Default:** `undefined`