diff --git a/.gitignore b/.gitignore index e6fecff012e2..67e90a1a3058 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ dist ltex* .DS_Store bench/test/*/*/ -**/benchmark/bench.json +**/bench.json **/browser/browser.json cypress/videos cypress/downloads diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 9d1c96a18fa3..26d82b7fa1c1 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -119,7 +119,8 @@ export function resolveConfig( if (resolved.minWorkers) resolved.minWorkers = Number(resolved.minWorkers) - resolved.fileParallelism ??= true + // run benchmark sequentially by default + resolved.fileParallelism ??= mode !== 'benchmark' if (!resolved.fileParallelism) { // ignore user config, parallelism cannot be implemented without limiting workers diff --git a/packages/vitest/src/runtime/runners/benchmark.ts b/packages/vitest/src/runtime/runners/benchmark.ts index 87838b007153..a72a6a232ef1 100644 --- a/packages/vitest/src/runtime/runners/benchmark.ts +++ b/packages/vitest/src/runtime/runners/benchmark.ts @@ -35,8 +35,9 @@ async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) { benchmarkSuiteGroup.push(task) } - if (benchmarkSuiteGroup.length) - await Promise.all(benchmarkSuiteGroup.map(subSuite => runBenchmarkSuite(subSuite, runner))) + // run sub suites sequentially + for (const subSuite of benchmarkSuiteGroup) + await runBenchmarkSuite(subSuite, runner) if (benchmarkGroup.length) { const defer = createDefer() diff --git a/test/benchmark/test/base.bench.ts b/test/benchmark/fixtures/basic/base.bench.ts similarity index 100% rename from test/benchmark/test/base.bench.ts rename to test/benchmark/fixtures/basic/base.bench.ts diff --git a/test/benchmark/test/mode.bench.ts b/test/benchmark/fixtures/basic/mode.bench.ts similarity index 100% rename from test/benchmark/test/mode.bench.ts rename to test/benchmark/fixtures/basic/mode.bench.ts diff --git a/test/benchmark/test/only.bench.ts b/test/benchmark/fixtures/basic/only.bench.ts similarity index 100% rename from test/benchmark/test/only.bench.ts rename to test/benchmark/fixtures/basic/only.bench.ts diff --git a/test/benchmark/fixtures/basic/vitest.config.ts b/test/benchmark/fixtures/basic/vitest.config.ts new file mode 100644 index 000000000000..abed6b2116e1 --- /dev/null +++ b/test/benchmark/fixtures/basic/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({}) diff --git a/test/benchmark/fixtures/sequential/f1.bench.ts b/test/benchmark/fixtures/sequential/f1.bench.ts new file mode 100644 index 000000000000..acf940fae47b --- /dev/null +++ b/test/benchmark/fixtures/sequential/f1.bench.ts @@ -0,0 +1,26 @@ +import { bench, describe } from "vitest" +import { appendLog, benchOptions, sleepBench } from "./helper"; + +bench("B1", async () => { + await appendLog("F1 / B1") + await sleepBench(); +}, benchOptions) + +describe("S1", () => { + bench("B1", async () => { + await appendLog("F1 / S1 / B1") + await sleepBench(); + }, benchOptions) + + bench("B2", async () => { + await appendLog("F1 / S1 / B2") + await sleepBench(); + }, benchOptions) +}) + +describe("S2", () => { + bench("B1", async () => { + await appendLog("F1 / S2 / B1") + await sleepBench(); + }, benchOptions) +}) diff --git a/test/benchmark/fixtures/sequential/f2.bench.ts b/test/benchmark/fixtures/sequential/f2.bench.ts new file mode 100644 index 000000000000..3968708f7a8b --- /dev/null +++ b/test/benchmark/fixtures/sequential/f2.bench.ts @@ -0,0 +1,9 @@ +import { bench, describe } from "vitest" +import { appendLog, benchOptions, sleepBench } from "./helper"; + +describe("S1", () => { + bench("B1", async () => { + await appendLog("F2 / S1 / B1") + await sleepBench(); + }, benchOptions) +}) diff --git a/test/benchmark/fixtures/sequential/helper.ts b/test/benchmark/fixtures/sequential/helper.ts new file mode 100644 index 000000000000..b609d7c9375c --- /dev/null +++ b/test/benchmark/fixtures/sequential/helper.ts @@ -0,0 +1,19 @@ +import fs from "node:fs"; + +const SLEEP_BENCH_MS = Number(process.env["SLEEP_BENCH_MS"] || 10); +const BENCH_ITERATIONS = Number(process.env["BENCH_ITERATIONS"] || 3); + +export const sleepBench = () => new Promise(resolve => setTimeout(resolve, SLEEP_BENCH_MS)) + +export const testLogFile = new URL("./test.log", import.meta.url); + +export async function appendLog(data: string) { + await fs.promises.appendFile(testLogFile, data + "\n"); +} + +export const benchOptions = { + time: 0, + iterations: BENCH_ITERATIONS, + warmupIterations: 0, + warmupTime: 0, +} diff --git a/test/benchmark/fixtures/sequential/setup.ts b/test/benchmark/fixtures/sequential/setup.ts new file mode 100644 index 000000000000..9a929fa54cad --- /dev/null +++ b/test/benchmark/fixtures/sequential/setup.ts @@ -0,0 +1,6 @@ +import fs from "node:fs"; +import { testLogFile } from "./helper"; + +export default async function setup() { + await fs.promises.rm(testLogFile, { force: true }); +} diff --git a/test/benchmark/fixtures/sequential/vitest.config.ts b/test/benchmark/fixtures/sequential/vitest.config.ts new file mode 100644 index 000000000000..fe63b73cf264 --- /dev/null +++ b/test/benchmark/fixtures/sequential/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config" + +// to see the difference better, increase sleep time and iterations e.g. by +// SLEEP_BENCH_MS=100 pnpm -C test/benchmark test bench -- --root fixtures/sequential --fileParallelism + +export default defineConfig({ + test: { + globalSetup: ["./setup.ts"] + } +}); diff --git a/test/benchmark/package.json b/test/benchmark/package.json index 2e72a0df4901..5566080b1da6 100644 --- a/test/benchmark/package.json +++ b/test/benchmark/package.json @@ -1,11 +1,9 @@ { - "name": "@vitest/benchmark", + "name": "@vitest/benchmark-sequential", "type": "module", "private": true, "scripts": { - "test": "node --test specs/* && echo '1'", - "bench:json": "vitest bench --reporter=json", - "bench": "vitest bench" + "test": "vitest" }, "devDependencies": { "vitest": "workspace:*" diff --git a/test/benchmark/specs/runner.test.mjs b/test/benchmark/specs/runner.test.mjs deleted file mode 100644 index 0a53b7ab05a4..000000000000 --- a/test/benchmark/specs/runner.test.mjs +++ /dev/null @@ -1,40 +0,0 @@ -import { existsSync, rmSync } from 'node:fs' -import * as assert from 'node:assert' -import { readFile } from 'node:fs/promises' -import test from 'node:test' -import { startVitest } from 'vitest/node' - -if (existsSync('./bench.json')) - rmSync('./bench.json') - -try { - await startVitest('benchmark', ['base.bench', 'mode.bench', 'only.bench'], { - watch: false, - }) -} -catch (error) { - console.error(error) - process.exit(1) -} - -const benchResult = await readFile('./bench.json', 'utf-8') -const resultJson = JSON.parse(benchResult) - -await test('benchmarks are actually running', async () => { - assert.ok(resultJson.testResults.sort, 'sort is in results') - assert.ok(resultJson.testResults.timeout, 'timeout is in results') - assert.ok(resultJson.testResults.a0, 'a0 is in results') - assert.ok(resultJson.testResults.c1, 'c1 is in results') - assert.ok(resultJson.testResults.a2, 'a2 is in results') - assert.ok(resultJson.testResults.b3, 'b3 is in results') - assert.ok(resultJson.testResults.b4, 'b4 is in results') -}) - -await test('doesn\'t have skipped tests', () => { - assert.doesNotMatch(benchResult, /skip/, 'contains skipped benchmarks') - - const skippedBenches = ['s0', 's1', 's2', 's3', 'sb4', 's4'] - const todoBenches = ['unimplemented suite', 'unimplemented test'] - - assert.ok(skippedBenches.concat(todoBenches).every(b => !benchResult.includes(b)), 'contains skipped benchmarks') -}) diff --git a/test/benchmark/test/__snapshots__/sequential.test.ts.snap b/test/benchmark/test/__snapshots__/sequential.test.ts.snap new file mode 100644 index 000000000000..6c7ec14b6d45 --- /dev/null +++ b/test/benchmark/test/__snapshots__/sequential.test.ts.snap @@ -0,0 +1,20 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`sequential 1`] = ` +"F1 / S1 / B1 +F1 / S1 / B1 +F1 / S1 / B1 +F1 / S1 / B2 +F1 / S1 / B2 +F1 / S1 / B2 +F1 / S2 / B1 +F1 / S2 / B1 +F1 / S2 / B1 +F1 / B1 +F1 / B1 +F1 / B1 +F2 / S1 / B1 +F2 / S1 / B1 +F2 / S1 / B1 +" +`; diff --git a/test/benchmark/test/basic.test.ts b/test/benchmark/test/basic.test.ts new file mode 100644 index 000000000000..46021e7483f4 --- /dev/null +++ b/test/benchmark/test/basic.test.ts @@ -0,0 +1,38 @@ +import fs from 'node:fs' +import { expect, it } from 'vitest' +import * as pathe from 'pathe' +import { runVitest } from '../../test-utils' + +it('basic', { timeout: 60_000 }, async () => { + const root = pathe.join(import.meta.dirname, '../fixtures/basic') + const benchFile = pathe.join(root, 'bench.json') + fs.rmSync(benchFile, { force: true }) + + await runVitest({ + root, + allowOnly: true, + benchmark: { + reporters: 'json', + outputFile: 'bench.json', + }, + }, [], 'benchmark') + + const benchResult = await fs.promises.readFile(benchFile, 'utf-8') + const resultJson = JSON.parse(benchResult) + + expect(Object.keys(resultJson.testResults)).toEqual( + expect.arrayContaining([ + 'sort', + 'timeout', + 'a0', + 'c1', + 'a2', + 'b3', + 'b4', + ]), + ) + + const skipped = ['skip', 's0', 's1', 's2', 's3', 'sb4', 's4', 'unimplemented suite', 'unimplemented test'] + for (const b of skipped) + expect(benchResult).not.toContain(b) +}) diff --git a/test/benchmark/test/sequential.test.ts b/test/benchmark/test/sequential.test.ts new file mode 100644 index 000000000000..4106d83498cc --- /dev/null +++ b/test/benchmark/test/sequential.test.ts @@ -0,0 +1,11 @@ +import fs from 'node:fs' +import { expect, it } from 'vitest' +import * as pathe from 'pathe' +import { runVitest } from '../../test-utils' + +it('sequential', async () => { + const root = pathe.join(import.meta.dirname, '../fixtures/sequential') + await runVitest({ root }, [], 'benchmark') + const testLog = await fs.promises.readFile(pathe.join(root, 'test.log'), 'utf-8') + expect(testLog).toMatchSnapshot() +}) diff --git a/test/benchmark/vitest.config.ts b/test/benchmark/vitest.config.ts index 790c4797b8e3..abed6b2116e1 100644 --- a/test/benchmark/vitest.config.ts +++ b/test/benchmark/vitest.config.ts @@ -1,25 +1,3 @@ import { defineConfig } from 'vitest/config' -function noop() {} - -export default defineConfig({ - test: { - update: false, - allowOnly: true, - benchmark: { - outputFile: './bench.json', - reporters: ['json', { - onInit: noop, - onPathsCollected: noop, - onCollected: noop, - onFinished: noop, - onTaskUpdate: noop, - onTestRemoved: noop, - onWatcherStart: noop, - onWatcherRerun: noop, - onServerRestart: noop, - onUserConsoleLog: noop, - }], - }, - }, -}) +export default defineConfig({})