From 7255061e7d884af7236511978800e0eb1ba15e5e Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Mon, 15 Aug 2022 15:01:44 +0800 Subject: [PATCH 1/5] chore: wip --- packages/vitest/package.json | 2 + packages/vitest/rollup.config.js | 1 + packages/vitest/src/node/cli-wrapper.ts | 86 +++++++++++++++++++++++++ pnpm-lock.yaml | 8 +++ 4 files changed, 97 insertions(+) create mode 100644 packages/vitest/src/node/cli-wrapper.ts diff --git a/packages/vitest/package.json b/packages/vitest/package.json index 0b823ba7821a..c91a893baea9 100644 --- a/packages/vitest/package.json +++ b/packages/vitest/package.json @@ -112,6 +112,7 @@ "@types/diff": "^5.0.2", "@types/jsdom": "^20.0.0", "@types/micromatch": "^4.0.2", + "@types/minimist": "^1.2.2", "@types/natural-compare": "^1.4.1", "@types/prompts": "^2.4.0", "@types/sinonjs__fake-timers": "^8.1.2", @@ -131,6 +132,7 @@ "log-update": "^5.0.1", "magic-string": "^0.26.2", "micromatch": "^4.0.5", + "minimist": "^1.2.6", "mlly": "^0.5.12", "natural-compare": "^1.4.0", "p-limit": "^4.0.0", diff --git a/packages/vitest/rollup.config.js b/packages/vitest/rollup.config.js index db5a61798aef..b4220dfcd3f2 100644 --- a/packages/vitest/rollup.config.js +++ b/packages/vitest/rollup.config.js @@ -18,6 +18,7 @@ const entries = [ 'src/index.ts', 'src/browser.ts', 'src/node/cli.ts', + 'src/node/cli-wrapper.ts', 'src/node.ts', 'src/runtime/worker.ts', 'src/runtime/loader.ts', diff --git a/packages/vitest/src/node/cli-wrapper.ts b/packages/vitest/src/node/cli-wrapper.ts new file mode 100644 index 000000000000..a7c1e62e4d89 --- /dev/null +++ b/packages/vitest/src/node/cli-wrapper.ts @@ -0,0 +1,86 @@ +/* eslint-disable no-console */ +import c from 'picocolors' +import minimist from 'minimist' +import { execaNode } from 'execa' + +const ENTRY = './cli.mjs' + +// Node errors seen in Vitest (vitejs/vite#9492) +const ERRORS = [ + 'Check failed: result.second.', // nodejs/node#43617 + 'FATAL ERROR: v8::FromJust Maybe value is Nothing.', // vitest-dev/vitest#1191 +] + +interface Args { + args: string[] + retries: number +} + +function parseArgs(): Args { + const args = minimist(process.argv.slice(2), { + 'string': ['segfault-retry'], + '--': true, + 'stopEarly': true, + }) + + const showUsageAndExit = (msg: string) => { + console.error(msg) + process.exit(1) + } + + if (args.r && Number.isNaN(Number(args.r))) + showUsageAndExit(c.red('Invalid value')) + + if (!args._.length) + showUsageAndExit(c.red('Missing argument')) + + return { + retries: Number(args.r), + args: args._, + } +} + +function findError(log: string) { + return log ? ERRORS.find(error => log.includes(error)) ?? '' : '' +} + +async function main({ args, retries }: Args) { + // default exit code = 100, as in retries were exhausted + let exitCode = 100 + + console.log(args) + + for (let i = 0; i < retries; i++) { + const childProc = execaNode(ENTRY, args, { + reject: false, + all: true, + }) + childProc.all!.pipe(process.stdout) + const { all: cmdOutput } = await childProc + + const error = findError(cmdOutput ?? '') + if (error) { + // use GitHub Action annotation to highlight error + if (process.env.GITHUB_ACTIONS) + console.log(`::warning::FLAKE DETECTED: ${error}`) + + console.log( + `${c.black(c.bgRed(' FLAKE DETECTED: ')) + } ${ + c.red(error)}`, + ) + console.log( + `${c.black(c.bgBlue(' RETRYING: '))} ${c.gray( + `(${i + 1} of ${retries})`, + )} ${c.blue(args.join(' '))}`, + ) + } + else { + exitCode = childProc.exitCode! + break + } + } + process.exit(exitCode) +} + +main(parseArgs()) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e5954c010477..05a9aa619095 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -694,6 +694,7 @@ importers: '@types/diff': ^5.0.2 '@types/jsdom': ^20.0.0 '@types/micromatch': ^4.0.2 + '@types/minimist': ^1.2.2 '@types/natural-compare': ^1.4.1 '@types/node': '*' '@types/prompts': ^2.4.0 @@ -717,6 +718,7 @@ importers: log-update: ^5.0.1 magic-string: ^0.26.2 micromatch: ^4.0.5 + minimist: ^1.2.6 mlly: ^0.5.12 natural-compare: ^1.4.0 p-limit: ^4.0.0 @@ -751,6 +753,7 @@ importers: '@types/diff': 5.0.2 '@types/jsdom': 20.0.0 '@types/micromatch': 4.0.2 + '@types/minimist': 1.2.2 '@types/natural-compare': 1.4.1 '@types/prompts': 2.4.0 '@types/sinonjs__fake-timers': 8.1.2 @@ -770,6 +773,7 @@ importers: log-update: 5.0.1 magic-string: 0.26.2 micromatch: 4.0.5 + minimist: 1.2.6 mlly: 0.5.12 natural-compare: 1.4.0 p-limit: 4.0.0 @@ -7444,6 +7448,10 @@ packages: resolution: {integrity: sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==} dev: true + /@types/minimist/1.2.2: + resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} + dev: true + /@types/ms/0.7.31: resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} dev: true From 93bb8a29b6e2dfd59b9814b2caa2ea98306c0845 Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Mon, 15 Aug 2022 15:10:26 +0800 Subject: [PATCH 2/5] chore: update --- packages/vitest/src/node/cli-wrapper.ts | 22 +++++++++++----------- packages/vitest/vitest.mjs | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/vitest/src/node/cli-wrapper.ts b/packages/vitest/src/node/cli-wrapper.ts index a7c1e62e4d89..3c4ca71c2f06 100644 --- a/packages/vitest/src/node/cli-wrapper.ts +++ b/packages/vitest/src/node/cli-wrapper.ts @@ -1,4 +1,7 @@ /* eslint-disable no-console */ +/** + * Wrapper of the CLI with child process to manage segfaults and retries. + */ import c from 'picocolors' import minimist from 'minimist' import { execaNode } from 'execa' @@ -17,8 +20,9 @@ interface Args { } function parseArgs(): Args { + const OPTION = 'segfault-retry' const args = minimist(process.argv.slice(2), { - 'string': ['segfault-retry'], + 'string': [OPTION], '--': true, 'stopEarly': true, }) @@ -28,15 +32,12 @@ function parseArgs(): Args { process.exit(1) } - if (args.r && Number.isNaN(Number(args.r))) - showUsageAndExit(c.red('Invalid value')) - - if (!args._.length) - showUsageAndExit(c.red('Missing argument')) + if (args.r && Number.isNaN(Number(args.OPTION))) + showUsageAndExit(c.red(`Invalid ${OPTION} value`)) return { - retries: Number(args.r), - args: args._, + retries: Number(args[OPTION]), + args: args._ || [], } } @@ -46,7 +47,7 @@ function findError(log: string) { async function main({ args, retries }: Args) { // default exit code = 100, as in retries were exhausted - let exitCode = 100 + const exitCode = 100 console.log(args) @@ -76,8 +77,7 @@ async function main({ args, retries }: Args) { ) } else { - exitCode = childProc.exitCode! - break + process.exit(childProc.exitCode!) } } process.exit(exitCode) diff --git a/packages/vitest/vitest.mjs b/packages/vitest/vitest.mjs index dfaf05f626b7..81939defef1c 100755 --- a/packages/vitest/vitest.mjs +++ b/packages/vitest/vitest.mjs @@ -1,2 +1,2 @@ #!/usr/bin/env node -import './dist/cli.mjs' +import './dist/cli-wrapper.mjs' From d1edf899c7b74d6b87ff3f39480f6c8aa1051df2 Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Mon, 15 Aug 2022 16:25:12 +0800 Subject: [PATCH 3/5] chore: feat update --- packages/vitest/src/node/cli-wrapper.ts | 132 +++++++++++++----------- packages/vitest/src/node/cli.ts | 1 + 2 files changed, 72 insertions(+), 61 deletions(-) diff --git a/packages/vitest/src/node/cli-wrapper.ts b/packages/vitest/src/node/cli-wrapper.ts index 3c4ca71c2f06..276c7b5a5557 100644 --- a/packages/vitest/src/node/cli-wrapper.ts +++ b/packages/vitest/src/node/cli-wrapper.ts @@ -2,85 +2,95 @@ /** * Wrapper of the CLI with child process to manage segfaults and retries. */ +import { fileURLToPath } from 'url' import c from 'picocolors' -import minimist from 'minimist' -import { execaNode } from 'execa' +import { execa } from 'execa' -const ENTRY = './cli.mjs' +const ENTRY = new URL('./cli.mjs', import.meta.url) + +interface ErrorDef { + trigger: string + url: string +} // Node errors seen in Vitest (vitejs/vite#9492) -const ERRORS = [ - 'Check failed: result.second.', // nodejs/node#43617 - 'FATAL ERROR: v8::FromJust Maybe value is Nothing.', // vitest-dev/vitest#1191 +const ERRORS: ErrorDef[] = [ + { + trigger: 'Check failed: result.second.', + url: 'https://github.com/nodejs/node/issues/43617', + }, + { + trigger: 'FATAL ERROR: v8::FromJust Maybe value is Nothing.', + url: 'https://github.com/vitest-dev/vitest/issues/1191', + }, + { + trigger: 'FATAL ERROR: v8::ToLocalChecked Empty MaybeLocal.', + url: 'https://github.com/nodejs/node/issues/42407', + }, ] -interface Args { - args: string[] - retries: number -} - -function parseArgs(): Args { - const OPTION = 'segfault-retry' - const args = minimist(process.argv.slice(2), { - 'string': [OPTION], - '--': true, - 'stopEarly': true, - }) +async function main() { + // default exit code = 100, as in retries were exhausted + const exitCode = 100 + let retries = 0 + const args = process.argv.slice(2) - const showUsageAndExit = (msg: string) => { - console.error(msg) - process.exit(1) + if (process.env.VITEST_SEGFAULT_RETRY) { + retries = +process.env.VITEST_SEGFAULT_RETRY + } + else { + for (let i = 0; i < args.length; i++) { + if (args[i].startsWith('--segfault-retry=')) { + retries = +args[i].split('=')[1] + break + } + else if (args[i] === '--segfault-retry' && args[i + 1]?.match(/^\d+$/)) { + retries = +args[i + 1] + break + } + } } - if (args.r && Number.isNaN(Number(args.OPTION))) - showUsageAndExit(c.red(`Invalid ${OPTION} value`)) + retries = Math.max(1, retries || 1) - return { - retries: Number(args[OPTION]), - args: args._ || [], + for (let i = 1; i <= retries; i++) { + if (i !== 1) + console.log(`${c.inverse(c.bold(c.magenta(' Retrying ')))} vitest ${args.join(' ')} ${c.gray(`(${i} of ${retries})`)}`) + await start(args) + if (i === 1 && retries === 1) { + console.log(c.yellow(`It seems to be an upstream bug of Node.js. To improve the test stability, +you could pass ${c.bold(c.green('--segfault-retry=3'))} or set env ${c.bold(c.green('VITEST_SEGFAULT_RETRY=3'))} to +have Vitest auto retries on flaky segfaults.\n`)) + } } + process.exit(exitCode) } -function findError(log: string) { - return log ? ERRORS.find(error => log.includes(error)) ?? '' : '' -} - -async function main({ args, retries }: Args) { - // default exit code = 100, as in retries were exhausted - const exitCode = 100 - - console.log(args) +main() - for (let i = 0; i < retries; i++) { - const childProc = execaNode(ENTRY, args, { - reject: false, - all: true, - }) - childProc.all!.pipe(process.stdout) - const { all: cmdOutput } = await childProc +async function start(args: string[]) { + const child = execa('node', [fileURLToPath(ENTRY), ...args], { + reject: false, + all: true, + stderr: 'pipe', + stdout: 'inherit', + stdin: 'inherit', + }) + child.stderr?.pipe(process.stderr) + const { all: output = '' } = await child - const error = findError(cmdOutput ?? '') - if (error) { - // use GitHub Action annotation to highlight error + for (const error of ERRORS) { + if (output.includes(error.trigger)) { if (process.env.GITHUB_ACTIONS) - console.log(`::warning::FLAKE DETECTED: ${error}`) + console.log(`::warning:: Segmentfault Error Detected: ${error.trigger}\nRefer to ${error.url}`) - console.log( - `${c.black(c.bgRed(' FLAKE DETECTED: ')) - } ${ - c.red(error)}`, - ) - console.log( - `${c.black(c.bgBlue(' RETRYING: '))} ${c.gray( - `(${i + 1} of ${retries})`, - )} ${c.blue(args.join(' '))}`, - ) - } - else { - process.exit(childProc.exitCode!) + const RED_BLOCK = c.inverse(c.red(' ')) + console.log(`\n${c.inverse(c.bold(c.red(' Segmentfault Error Detected ')))}\n${RED_BLOCK} ${c.red(error.trigger)}\n${RED_BLOCK} ${c.red(`Refer to ${error.url}`)}\n`) + return } } - process.exit(exitCode) + + // no segmentfault found + process.exit(child.exitCode!) } -main(parseArgs()) diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index 470f5e2d6d55..988f900ca2ab 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -39,6 +39,7 @@ cli .option('--changed [since]', 'Run tests that are affected by the changed files (default: false)') .option('--sequence ', 'Define in what order to run tests (use --sequence.shuffle to run tests in random order)') .option('--no-color', 'Removes colors from the console output') + .option('--segfault-retry ', 'Return tests on segment fault (default: 0)', { default: 0 }) .help() cli From 367be72863a605e71e473fb058223153667290e3 Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Mon, 15 Aug 2022 16:31:08 +0800 Subject: [PATCH 4/5] chore: cleanup package --- packages/vitest/package.json | 2 -- pnpm-lock.yaml | 8 -------- 2 files changed, 10 deletions(-) diff --git a/packages/vitest/package.json b/packages/vitest/package.json index f5454da16f1f..146639cdf925 100644 --- a/packages/vitest/package.json +++ b/packages/vitest/package.json @@ -112,7 +112,6 @@ "@types/diff": "^5.0.2", "@types/jsdom": "^20.0.0", "@types/micromatch": "^4.0.2", - "@types/minimist": "^1.2.2", "@types/natural-compare": "^1.4.1", "@types/prompts": "^2.4.0", "@types/sinonjs__fake-timers": "^8.1.2", @@ -132,7 +131,6 @@ "log-update": "^5.0.1", "magic-string": "^0.26.2", "micromatch": "^4.0.5", - "minimist": "^1.2.6", "mlly": "^0.5.12", "natural-compare": "^1.4.0", "p-limit": "^4.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 25f3fba6c926..5d25f7c0d66c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -702,7 +702,6 @@ importers: '@types/diff': ^5.0.2 '@types/jsdom': ^20.0.0 '@types/micromatch': ^4.0.2 - '@types/minimist': ^1.2.2 '@types/natural-compare': ^1.4.1 '@types/node': '*' '@types/prompts': ^2.4.0 @@ -726,7 +725,6 @@ importers: log-update: ^5.0.1 magic-string: ^0.26.2 micromatch: ^4.0.5 - minimist: ^1.2.6 mlly: ^0.5.12 natural-compare: ^1.4.0 p-limit: ^4.0.0 @@ -761,7 +759,6 @@ importers: '@types/diff': 5.0.2 '@types/jsdom': 20.0.0 '@types/micromatch': 4.0.2 - '@types/minimist': 1.2.2 '@types/natural-compare': 1.4.1 '@types/prompts': 2.4.0 '@types/sinonjs__fake-timers': 8.1.2 @@ -781,7 +778,6 @@ importers: log-update: 5.0.1 magic-string: 0.26.2 micromatch: 4.0.5 - minimist: 1.2.6 mlly: 0.5.12 natural-compare: 1.4.0 p-limit: 4.0.0 @@ -7235,10 +7231,6 @@ packages: resolution: {integrity: sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==} dev: true - /@types/minimist/1.2.2: - resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} - dev: true - /@types/ms/0.7.31: resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} dev: true From c88cf3971b6a04147f95d87c025bd1dcbd19f977 Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Mon, 15 Aug 2022 16:32:01 +0800 Subject: [PATCH 5/5] chore: retry on ci --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab4a9fbe588b..25ade44a94c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,9 @@ on: branches: - main +env: + VITEST_SEGFAULT_RETRY: 3 + jobs: lint: runs-on: ubuntu-latest