diff --git a/packages/expect/package.json b/packages/expect/package.json index f8d93a553eff..7425f8d02c75 100644 --- a/packages/expect/package.json +++ b/packages/expect/package.json @@ -39,6 +39,7 @@ "chai": "^4.3.7" }, "devDependencies": { + "@vitest/runner": "workspace:*", "picocolors": "^1.0.0" } } diff --git a/packages/expect/rollup.config.js b/packages/expect/rollup.config.js index 93f60d59c401..6ba38766cce2 100644 --- a/packages/expect/rollup.config.js +++ b/packages/expect/rollup.config.js @@ -9,6 +9,7 @@ const external = [ ...Object.keys(pkg.dependencies || {}), ...Object.keys(pkg.peerDependencies || {}), '@vitest/utils/diff', + '@vitest/utils/error', ] const plugins = [ diff --git a/packages/expect/src/jest-expect.ts b/packages/expect/src/jest-expect.ts index 424554546c14..0ad8ced8c88b 100644 --- a/packages/expect/src/jest-expect.ts +++ b/packages/expect/src/jest-expect.ts @@ -3,12 +3,13 @@ import { assertTypes, getColors } from '@vitest/utils' import type { Constructable } from '@vitest/utils' import type { EnhancedSpy } from '@vitest/spy' import { isMockFunction } from '@vitest/spy' +import type { Test } from '@vitest/runner' import type { Assertion, ChaiPlugin } from './types' import { arrayBufferEquality, generateToBeMessage, iterableEquality, equals as jestEquals, sparseArrayEquality, subsetEquality, typeEquality } from './jest-utils' import type { AsymmetricMatcher } from './jest-asymmetric-matchers' import { diff, stringify } from './jest-matcher-utils' import { JEST_MATCHERS_OBJECT } from './constants' -import { recordAsyncExpect } from './utils' +import { recordAsyncExpect, wrapSoft } from './utils' // Jest Expect Compact export const JestChaiExpect: ChaiPlugin = (chai, utils) => { @@ -16,8 +17,9 @@ export const JestChaiExpect: ChaiPlugin = (chai, utils) => { function def(name: keyof Assertion | (keyof Assertion)[], fn: ((this: Chai.AssertionStatic & Assertion, ...args: any[]) => any)) { const addMethod = (n: keyof Assertion) => { - utils.addMethod(chai.Assertion.prototype, n, fn) - utils.addMethod((globalThis as any)[JEST_MATCHERS_OBJECT].matchers, n, fn) + const softWrapper = wrapSoft(utils, fn) + utils.addMethod(chai.Assertion.prototype, n, softWrapper) + utils.addMethod((globalThis as any)[JEST_MATCHERS_OBJECT].matchers, n, softWrapper) } if (Array.isArray(name)) @@ -636,7 +638,7 @@ export const JestChaiExpect: ChaiPlugin = (chai, utils) => { utils.addProperty(chai.Assertion.prototype, 'resolves', function __VITEST_RESOLVES__(this: any) { utils.flag(this, 'promise', 'resolves') utils.flag(this, 'error', new Error('resolves')) - const test = utils.flag(this, 'vitest-test') + const test: Test = utils.flag(this, 'vitest-test') const obj = utils.flag(this, 'object') if (typeof obj?.then !== 'function') @@ -671,7 +673,7 @@ export const JestChaiExpect: ChaiPlugin = (chai, utils) => { utils.addProperty(chai.Assertion.prototype, 'rejects', function __VITEST_REJECTS__(this: any) { utils.flag(this, 'promise', 'rejects') utils.flag(this, 'error', new Error('rejects')) - const test = utils.flag(this, 'vitest-test') + const test: Test = utils.flag(this, 'vitest-test') const obj = utils.flag(this, 'object') const wrapper = typeof obj === 'function' ? obj() : obj // for jest compat diff --git a/packages/expect/src/jest-extend.ts b/packages/expect/src/jest-extend.ts index 4fd9e6378033..f6160c63fb83 100644 --- a/packages/expect/src/jest-extend.ts +++ b/packages/expect/src/jest-extend.ts @@ -17,6 +17,7 @@ import { iterableEquality, subsetEquality, } from './jest-utils' +import { wrapSoft } from './utils' function getMatcherState(assertion: Chai.AssertionStatic & Chai.Assertion, expect: ExpectStatic) { const obj = assertion._obj @@ -75,8 +76,9 @@ function JestExtendPlugin(expect: ExpectStatic, matchers: MatchersObject): ChaiP throw new JestExtendError(message(), actual, expected) } - utils.addMethod((globalThis as any)[JEST_MATCHERS_OBJECT].matchers, expectAssertionName, expectWrapper) - utils.addMethod(c.Assertion.prototype, expectAssertionName, expectWrapper) + const softWrapper = wrapSoft(utils, expectWrapper) + utils.addMethod((globalThis as any)[JEST_MATCHERS_OBJECT].matchers, expectAssertionName, softWrapper) + utils.addMethod(c.Assertion.prototype, expectAssertionName, softWrapper) class CustomMatcher extends AsymmetricMatcher<[unknown, ...unknown[]]> { constructor(inverse = false, ...sample: [unknown, ...unknown[]]) { diff --git a/packages/expect/src/types.ts b/packages/expect/src/types.ts index 78d5d0533166..6c4406bac208 100644 --- a/packages/expect/src/types.ts +++ b/packages/expect/src/types.ts @@ -79,6 +79,7 @@ export interface MatcherState { iterableEquality: Tester subsetEquality: Tester } + soft?: boolean } export interface SyncExpectationResult { @@ -100,7 +101,7 @@ export type MatchersObject = Record(actual: T, message?: string): Assertion - + soft(actual: T, message?: string): Assertion extend(expects: MatchersObject): void assertions(expected: number): void hasAssertions(): void diff --git a/packages/expect/src/utils.ts b/packages/expect/src/utils.ts index 1201f5a90b12..aa1754d57f78 100644 --- a/packages/expect/src/utils.ts +++ b/packages/expect/src/utils.ts @@ -1,3 +1,9 @@ +import { processError } from '@vitest/utils/error' +import type { Test } from '@vitest/runner/types' +import { GLOBAL_EXPECT } from './constants' +import { getState } from './state' +import type { Assertion, MatcherState } from './types' + export function recordAsyncExpect(test: any, promise: Promise | PromiseLike) { // record promise for test, that resolves before test ends if (test && promise instanceof Promise) { @@ -16,3 +22,30 @@ export function recordAsyncExpect(test: any, promise: Promise | PromiseLike return promise } + +export function wrapSoft(utils: Chai.ChaiUtils, fn: (this: Chai.AssertionStatic & Assertion, ...args: any[]) => void) { + return function (this: Chai.AssertionStatic & Assertion, ...args: any[]) { + const test: Test = utils.flag(this, 'vitest-test') + + // @ts-expect-error local is untyped + const state: MatcherState = test?.context._local + ? test.context.expect.getState() + : getState((globalThis as any)[GLOBAL_EXPECT]) + + if (!state.soft) + return fn.apply(this, args) + + if (!test) + throw new Error('expect.soft() can only be used inside a test') + + try { + return fn.apply(this, args) + } + catch (err) { + test.result ||= { state: 'fail' } + test.result.state = 'fail' + test.result.errors ||= [] + test.result.errors.push(processError(err)) + } + } +} diff --git a/packages/runner/rollup.config.js b/packages/runner/rollup.config.js index 1d1f02aca174..cebc5be6478a 100644 --- a/packages/runner/rollup.config.js +++ b/packages/runner/rollup.config.js @@ -9,7 +9,7 @@ const external = [ ...builtinModules, ...Object.keys(pkg.dependencies || {}), ...Object.keys(pkg.peerDependencies || {}), - '@vitest/utils/diff', + '@vitest/utils/error', ] const entries = { diff --git a/packages/runner/src/collect.ts b/packages/runner/src/collect.ts index 656b600baeba..807ca6fab0c4 100644 --- a/packages/runner/src/collect.ts +++ b/packages/runner/src/collect.ts @@ -1,10 +1,10 @@ import { relative } from 'pathe' +import { processError } from '@vitest/utils/error' import type { File } from './types' import type { VitestRunner } from './types/runner' import { calculateSuiteHash, generateHash, interpretTaskModes, someTasksAreOnly } from './utils/collect' import { clearCollectorContext, getDefaultSuite } from './suite' import { getHooks, setHooks } from './map' -import { processError } from './utils/error' import { collectorContext } from './context' import { runSetupFiles } from './setup' diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 70e9e75e7cad..1788675632e6 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -1,11 +1,11 @@ import limit from 'p-limit' import { getSafeTimers, shuffle } from '@vitest/utils' +import { processError } from '@vitest/utils/error' import type { VitestRunner } from './types/runner' import type { File, HookCleanupCallback, HookListener, SequenceHooks, Suite, SuiteHooks, Task, TaskMeta, TaskResult, TaskResultPack, TaskState, Test } from './types' import { partitionSuiteChildren } from './utils/suite' import { getFn, getHooks } from './map' import { collectTests } from './collect' -import { processError } from './utils/error' import { setCurrentTest } from './test-state' import { hasFailed, hasTests } from './utils/tasks' @@ -156,7 +156,6 @@ export async function runTest(test: Test, runner: VitestRunner) { throw new Error('Test function is not found. Did you add it using `setFn`?') await fn() } - // some async expect will be added to this array, in case user forget to await theme if (test.promises) { const result = await Promise.allSettled(test.promises) @@ -167,10 +166,12 @@ export async function runTest(test: Test, runner: VitestRunner) { await runner.onAfterTryTest?.(test, { retry: retryCount, repeats: repeatCount }) - if (!test.repeats) - test.result.state = 'pass' - else if (test.repeats && retry === retryCount) - test.result.state = 'pass' + if (test.result.state !== 'fail') { + if (!test.repeats) + test.result.state = 'pass' + else if (test.repeats && retry === retryCount) + test.result.state = 'pass' + } } catch (e) { failTask(test.result, e) @@ -186,6 +187,12 @@ export async function runTest(test: Test, runner: VitestRunner) { if (test.result.state === 'pass') break + + if (retryCount < retry - 1) { + // reset state when retry test + test.result.state = 'run' + } + // update retry info updateTask(test, runner) } diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index eeca49956da7..f2f745a6bf2a 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -1,6 +1,5 @@ -import type { Awaitable } from '@vitest/utils' +import type { Awaitable, ErrorWithDiff } from '@vitest/utils' import type { ChainableFunction } from '../utils/chain' -import type { ErrorWithDiff } from '../utils/error' export type RunMode = 'run' | 'skip' | 'only' | 'todo' export type TaskState = RunMode | 'pass' | 'fail' diff --git a/packages/runner/src/utils/collect.ts b/packages/runner/src/utils/collect.ts index 013faa8db3ad..b163bae7f716 100644 --- a/packages/runner/src/utils/collect.ts +++ b/packages/runner/src/utils/collect.ts @@ -1,5 +1,5 @@ +import { processError } from '@vitest/utils/error' import type { Suite, TaskBase } from '../types' -import { processError } from './error' /** * If any tasks been marked as `only`, mark all other tasks as `skip`. diff --git a/packages/runner/src/utils/error.ts b/packages/runner/src/utils/error.ts deleted file mode 100644 index 45891976f8dc..000000000000 --- a/packages/runner/src/utils/error.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { deepClone, format, getOwnProperties, getType, stringify } from '@vitest/utils' -import type { DiffOptions } from '@vitest/utils/diff' -import { unifiedDiff } from '@vitest/utils/diff' - -export type { ParsedStack, ErrorWithDiff } from '@vitest/utils' - -const IS_RECORD_SYMBOL = '@@__IMMUTABLE_RECORD__@@' -const IS_COLLECTION_SYMBOL = '@@__IMMUTABLE_ITERABLE__@@' - -function isImmutable(v: any) { - return v && (v[IS_COLLECTION_SYMBOL] || v[IS_RECORD_SYMBOL]) -} - -const OBJECT_PROTO = Object.getPrototypeOf({}) - -function getUnserializableMessage(err: unknown) { - if (err instanceof Error) - return `: ${err.message}` - if (typeof err === 'string') - return `: ${err}` - return '' -} - -// https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm -export function serializeError(val: any, seen = new WeakMap()): any { - if (!val || typeof val === 'string') - return val - if (typeof val === 'function') - return `Function<${val.name || 'anonymous'}>` - if (typeof val === 'symbol') - return val.toString() - if (typeof val !== 'object') - return val - // cannot serialize immutables as immutables - if (isImmutable(val)) - return serializeError(val.toJSON(), seen) - if (val instanceof Promise || (val.constructor && val.constructor.prototype === 'AsyncFunction')) - return 'Promise' - if (typeof Element !== 'undefined' && val instanceof Element) - return val.tagName - if (typeof val.asymmetricMatch === 'function') - return `${val.toString()} ${format(val.sample)}` - - if (seen.has(val)) - return seen.get(val) - - if (Array.isArray(val)) { - const clone: any[] = new Array(val.length) - seen.set(val, clone) - val.forEach((e, i) => { - try { - clone[i] = serializeError(e, seen) - } - catch (err) { - clone[i] = getUnserializableMessage(err) - } - }) - return clone - } - else { - // Objects with `Error` constructors appear to cause problems during worker communication - // using `MessagePort`, so the serialized error object is being recreated as plain object. - const clone = Object.create(null) - seen.set(val, clone) - - let obj = val - while (obj && obj !== OBJECT_PROTO) { - Object.getOwnPropertyNames(obj).forEach((key) => { - if (key in clone) - return - try { - clone[key] = serializeError(val[key], seen) - } - catch (err) { - // delete in case it has a setter from prototype that might throw - delete clone[key] - clone[key] = getUnserializableMessage(err) - } - }) - obj = Object.getPrototypeOf(obj) - } - return clone - } -} - -function normalizeErrorMessage(message: string) { - return message.replace(/__vite_ssr_import_\d+__\./g, '') -} - -export function processError(err: any, options: DiffOptions = {}) { - if (!err || typeof err !== 'object') - return { message: err } - // stack is not serialized in worker communication - // we stringify it first - if (err.stack) - err.stackStr = String(err.stack) - if (err.name) - err.nameStr = String(err.name) - - const clonedActual = deepClone(err.actual, { forceWritable: true }) - const clonedExpected = deepClone(err.expected, { forceWritable: true }) - - const { replacedActual, replacedExpected } = replaceAsymmetricMatcher(clonedActual, clonedExpected) - - if (err.showDiff || (err.showDiff === undefined && err.expected !== undefined && err.actual !== undefined)) - err.diff = unifiedDiff(replacedActual, replacedExpected, options) - - if (typeof err.expected !== 'string') - err.expected = stringify(err.expected, 10) - if (typeof err.actual !== 'string') - err.actual = stringify(err.actual, 10) - - // some Error implementations don't allow rewriting message - try { - if (typeof err.message === 'string') - err.message = normalizeErrorMessage(err.message) - - if (typeof err.cause === 'object' && typeof err.cause.message === 'string') - err.cause.message = normalizeErrorMessage(err.cause.message) - } - catch {} - - try { - return serializeError(err) - } - catch (e: any) { - return serializeError(new Error(`Failed to fully serialize error: ${e?.message}\nInner error message: ${err?.message}`)) - } -} - -function isAsymmetricMatcher(data: any) { - const type = getType(data) - return type === 'Object' && typeof data.asymmetricMatch === 'function' -} - -function isReplaceable(obj1: any, obj2: any) { - const obj1Type = getType(obj1) - const obj2Type = getType(obj2) - return obj1Type === obj2Type && obj1Type === 'Object' -} - -export function replaceAsymmetricMatcher(actual: any, expected: any, actualReplaced = new WeakSet(), expectedReplaced = new WeakSet()) { - if (!isReplaceable(actual, expected)) - return { replacedActual: actual, replacedExpected: expected } - if (actualReplaced.has(actual) || expectedReplaced.has(expected)) - return { replacedActual: actual, replacedExpected: expected } - actualReplaced.add(actual) - expectedReplaced.add(expected) - getOwnProperties(expected).forEach((key) => { - const expectedValue = expected[key] - const actualValue = actual[key] - if (isAsymmetricMatcher(expectedValue)) { - if (expectedValue.asymmetricMatch(actualValue)) - actual[key] = expectedValue - } - else if (isAsymmetricMatcher(actualValue)) { - if (actualValue.asymmetricMatch(expectedValue)) - expected[key] = actualValue - } - else if (isReplaceable(actualValue, expectedValue)) { - const replaced = replaceAsymmetricMatcher( - actualValue, - expectedValue, - actualReplaced, - expectedReplaced, - ) - actual[key] = replaced.replacedActual - expected[key] = replaced.replacedExpected - } - }) - return { - replacedActual: actual, - replacedExpected: expected, - } -} diff --git a/packages/runner/src/utils/index.ts b/packages/runner/src/utils/index.ts index ebbe97f11de4..46b2d49e5a38 100644 --- a/packages/runner/src/utils/index.ts +++ b/packages/runner/src/utils/index.ts @@ -2,4 +2,3 @@ export * from './collect' export * from './suite' export * from './tasks' export * from './chain' -export * from './error' diff --git a/packages/utils/error.d.ts b/packages/utils/error.d.ts new file mode 100644 index 000000000000..9329baa87d94 --- /dev/null +++ b/packages/utils/error.d.ts @@ -0,0 +1 @@ +export * from './dist/error.js' diff --git a/packages/utils/package.json b/packages/utils/package.json index 292ed2706ef8..3da3d7cda077 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -24,6 +24,10 @@ "types": "./dist/diff.d.ts", "import": "./dist/diff.js" }, + "./error": { + "types": "./dist/error.d.ts", + "import": "./dist/error.js" + }, "./helpers": { "types": "./dist/helpers.d.ts", "import": "./dist/helpers.js" diff --git a/packages/utils/rollup.config.js b/packages/utils/rollup.config.js index 2a315c966703..721ece68bdb9 100644 --- a/packages/utils/rollup.config.js +++ b/packages/utils/rollup.config.js @@ -10,6 +10,7 @@ const entries = { index: 'src/index.ts', helpers: 'src/helpers.ts', diff: 'src/diff.ts', + error: 'src/error.ts', types: 'src/types.ts', } diff --git a/packages/utils/src/base.ts b/packages/utils/src/base.ts new file mode 100644 index 000000000000..27ab8298929d --- /dev/null +++ b/packages/utils/src/base.ts @@ -0,0 +1,21 @@ +interface ErrorOptions { + message?: string + stackTraceLimit?: number +} +/** + * Get original stacktrace without source map support the most performant way. + * - Create only 1 stack frame. + * - Rewrite prepareStackTrace to bypass "support-stack-trace" (usually takes ~250ms). + */ +export function createSimpleStackTrace(options?: ErrorOptions) { + const { message = 'error', stackTraceLimit = 1 } = options || {} + const limit = Error.stackTraceLimit + const prepareStackTrace = Error.prepareStackTrace + Error.stackTraceLimit = stackTraceLimit + Error.prepareStackTrace = e => e.stack + const err = new Error(message) + const stackTrace = err.stack || '' + Error.prepareStackTrace = prepareStackTrace + Error.stackTraceLimit = limit + return stackTrace +} diff --git a/packages/utils/src/error.ts b/packages/utils/src/error.ts index 8189d03c7239..a610477688ec 100644 --- a/packages/utils/src/error.ts +++ b/packages/utils/src/error.ts @@ -1,22 +1,175 @@ -interface ErrorOptions { - message?: string - stackTraceLimit?: number -} - -/** - * Get original stacktrace without source map support the most performant way. - * - Create only 1 stack frame. - * - Rewrite prepareStackTrace to bypass "support-stack-trace" (usually takes ~250ms). - */ -export function createSimpleStackTrace(options?: ErrorOptions) { - const { message = 'error', stackTraceLimit = 1 } = options || {} - const limit = Error.stackTraceLimit - const prepareStackTrace = Error.prepareStackTrace - Error.stackTraceLimit = stackTraceLimit - Error.prepareStackTrace = e => e.stack - const err = new Error(message) - const stackTrace = err.stack || '' - Error.prepareStackTrace = prepareStackTrace - Error.stackTraceLimit = limit - return stackTrace +import type { DiffOptions } from './diff' +import { unifiedDiff } from './diff' +import { format } from './display' +import { deepClone, getOwnProperties, getType } from './helpers' +import { stringify } from './stringify' + +const IS_RECORD_SYMBOL = '@@__IMMUTABLE_RECORD__@@' +const IS_COLLECTION_SYMBOL = '@@__IMMUTABLE_ITERABLE__@@' + +function isImmutable(v: any) { + return v && (v[IS_COLLECTION_SYMBOL] || v[IS_RECORD_SYMBOL]) +} + +const OBJECT_PROTO = Object.getPrototypeOf({}) + +function getUnserializableMessage(err: unknown) { + if (err instanceof Error) + return `: ${err.message}` + if (typeof err === 'string') + return `: ${err}` + return '' +} + +// https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm +export function serializeError(val: any, seen = new WeakMap()): any { + if (!val || typeof val === 'string') + return val + if (typeof val === 'function') + return `Function<${val.name || 'anonymous'}>` + if (typeof val === 'symbol') + return val.toString() + if (typeof val !== 'object') + return val + // cannot serialize immutables as immutables + if (isImmutable(val)) + return serializeError(val.toJSON(), seen) + if (val instanceof Promise || (val.constructor && val.constructor.prototype === 'AsyncFunction')) + return 'Promise' + if (typeof Element !== 'undefined' && val instanceof Element) + return val.tagName + if (typeof val.asymmetricMatch === 'function') + return `${val.toString()} ${format(val.sample)}` + + if (seen.has(val)) + return seen.get(val) + + if (Array.isArray(val)) { + const clone: any[] = new Array(val.length) + seen.set(val, clone) + val.forEach((e, i) => { + try { + clone[i] = serializeError(e, seen) + } + catch (err) { + clone[i] = getUnserializableMessage(err) + } + }) + return clone + } + else { + // Objects with `Error` constructors appear to cause problems during worker communication + // using `MessagePort`, so the serialized error object is being recreated as plain object. + const clone = Object.create(null) + seen.set(val, clone) + + let obj = val + while (obj && obj !== OBJECT_PROTO) { + Object.getOwnPropertyNames(obj).forEach((key) => { + if (key in clone) + return + try { + clone[key] = serializeError(val[key], seen) + } + catch (err) { + // delete in case it has a setter from prototype that might throw + delete clone[key] + clone[key] = getUnserializableMessage(err) + } + }) + obj = Object.getPrototypeOf(obj) + } + return clone + } +} + +function normalizeErrorMessage(message: string) { + return message.replace(/__vite_ssr_import_\d+__\./g, '') +} + +export function processError(err: any, options: DiffOptions = {}) { + if (!err || typeof err !== 'object') + return { message: err } + // stack is not serialized in worker communication + // we stringify it first + if (err.stack) + err.stackStr = String(err.stack) + if (err.name) + err.nameStr = String(err.name) + + const clonedActual = deepClone(err.actual, { forceWritable: true }) + const clonedExpected = deepClone(err.expected, { forceWritable: true }) + + const { replacedActual, replacedExpected } = replaceAsymmetricMatcher(clonedActual, clonedExpected) + + if (err.showDiff || (err.showDiff === undefined && err.expected !== undefined && err.actual !== undefined)) + err.diff = unifiedDiff(replacedActual, replacedExpected, options) + + if (typeof err.expected !== 'string') + err.expected = stringify(err.expected, 10) + if (typeof err.actual !== 'string') + err.actual = stringify(err.actual, 10) + + // some Error implementations don't allow rewriting message + try { + if (typeof err.message === 'string') + err.message = normalizeErrorMessage(err.message) + + if (typeof err.cause === 'object' && typeof err.cause.message === 'string') + err.cause.message = normalizeErrorMessage(err.cause.message) + } + catch {} + + try { + return serializeError(err) + } + catch (e: any) { + return serializeError(new Error(`Failed to fully serialize error: ${e?.message}\nInner error message: ${err?.message}`)) + } +} + +function isAsymmetricMatcher(data: any) { + const type = getType(data) + return type === 'Object' && typeof data.asymmetricMatch === 'function' +} + +function isReplaceable(obj1: any, obj2: any) { + const obj1Type = getType(obj1) + const obj2Type = getType(obj2) + return obj1Type === obj2Type && obj1Type === 'Object' +} + +export function replaceAsymmetricMatcher(actual: any, expected: any, actualReplaced = new WeakSet(), expectedReplaced = new WeakSet()) { + if (!isReplaceable(actual, expected)) + return { replacedActual: actual, replacedExpected: expected } + if (actualReplaced.has(actual) || expectedReplaced.has(expected)) + return { replacedActual: actual, replacedExpected: expected } + actualReplaced.add(actual) + expectedReplaced.add(expected) + getOwnProperties(expected).forEach((key) => { + const expectedValue = expected[key] + const actualValue = actual[key] + if (isAsymmetricMatcher(expectedValue)) { + if (expectedValue.asymmetricMatch(actualValue)) + actual[key] = expectedValue + } + else if (isAsymmetricMatcher(actualValue)) { + if (actualValue.asymmetricMatch(expectedValue)) + expected[key] = actualValue + } + else if (isReplaceable(actualValue, expectedValue)) { + const replaced = replaceAsymmetricMatcher( + actualValue, + expectedValue, + actualReplaced, + expectedReplaced, + ) + actual[key] = replaced.replacedActual + expected[key] = replaced.replacedExpected + } + }) + return { + replacedActual: actual, + replacedExpected: expected, + } } diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index f52bac3f9414..cfe390aebc70 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -6,5 +6,5 @@ export * from './random' export * from './display' export * from './constants' export * from './colors' -export * from './error' +export * from './base' export * from './source-map' diff --git a/packages/vitest/rollup.config.js b/packages/vitest/rollup.config.js index 4b7f1451c87c..63d3cb65766b 100644 --- a/packages/vitest/rollup.config.js +++ b/packages/vitest/rollup.config.js @@ -61,6 +61,7 @@ const external = [ 'vite-node/server', 'vite-node/utils', '@vitest/utils/diff', + '@vitest/utils/error', '@vitest/runner/utils', '@vitest/runner/types', '@vitest/snapshot/environment', diff --git a/packages/vitest/src/integrations/chai/index.ts b/packages/vitest/src/integrations/chai/index.ts index 01ed3d84d22d..941374d19e97 100644 --- a/packages/vitest/src/integrations/chai/index.ts +++ b/packages/vitest/src/integrations/chai/index.ts @@ -12,7 +12,7 @@ import { getCurrentEnvironment, getFullName } from '../../utils' export function createExpect(test?: Test) { const expect = ((value: any, message?: string): Assertion => { const { assertionCalls } = getState(expect) - setState({ assertionCalls: assertionCalls + 1 }, expect) + setState({ assertionCalls: assertionCalls + 1, soft: false }, expect) const assert = chai.expect(value, message) as unknown as Assertion const _test = test || getCurrentTest() if (_test) @@ -45,6 +45,14 @@ export function createExpect(test?: Test) { // @ts-expect-error untyped expect.extend = matchers => chai.expect.extend(expect, matchers) + expect.soft = (...args) => { + const assert = expect(...args) + expect.setState({ + soft: true, + }) + return assert + } + function assertions(expected: number) { const errorGen = () => new Error(`expected number of assertions to be ${expected}, but got ${expect.getState().assertionCalls}`) if (Error.captureStackTrace) diff --git a/packages/vitest/src/node/reporters/junit.ts b/packages/vitest/src/node/reporters/junit.ts index 9252aac524c6..e0fccb84fea2 100644 --- a/packages/vitest/src/node/reporters/junit.ts +++ b/packages/vitest/src/node/reporters/junit.ts @@ -3,7 +3,7 @@ import { hostname } from 'node:os' import { dirname, relative, resolve } from 'pathe' import type { Task } from '@vitest/runner' -import type { ErrorWithDiff } from '@vitest/runner/utils' +import type { ErrorWithDiff } from '@vitest/utils' import type { Vitest } from '../../node' import type { Reporter } from '../../types/reporter' import { parseErrorStacktrace } from '../../utils/source-map' diff --git a/packages/vitest/src/node/reporters/tap.ts b/packages/vitest/src/node/reporters/tap.ts index 31de7864072d..d37538c3e271 100644 --- a/packages/vitest/src/node/reporters/tap.ts +++ b/packages/vitest/src/node/reporters/tap.ts @@ -1,5 +1,5 @@ import type { Task } from '@vitest/runner' -import type { ParsedStack } from '@vitest/runner/utils' +import type { ParsedStack } from '@vitest/utils' import type { Vitest } from '../../node' import type { Reporter } from '../../types/reporter' import { parseErrorStacktrace } from '../../utils/source-map' diff --git a/packages/vitest/src/runtime/execute.ts b/packages/vitest/src/runtime/execute.ts index bacd3c3b1747..3cc62ad2aa79 100644 --- a/packages/vitest/src/runtime/execute.ts +++ b/packages/vitest/src/runtime/execute.ts @@ -3,7 +3,7 @@ import { ModuleCacheMap, ViteNodeRunner } from 'vite-node/client' import { isInternalRequest, isNodeBuiltin, isPrimitive } from 'vite-node/utils' import type { ViteNodeRunnerOptions } from 'vite-node' import { normalize, relative, resolve } from 'pathe' -import { processError } from '@vitest/runner/utils' +import { processError } from '@vitest/utils/error' import type { MockMap } from '../types/mocker' import { getCurrentEnvironment, getWorkerState } from '../utils/global' import type { ContextRPC, ContextTestEnvironment, ResolvedConfig } from '../types' diff --git a/packages/vitest/src/types/general.ts b/packages/vitest/src/types/general.ts index e71d3c3ee3a5..1af33776a435 100644 --- a/packages/vitest/src/types/general.ts +++ b/packages/vitest/src/types/general.ts @@ -1,4 +1,4 @@ -export type { ErrorWithDiff, ParsedStack } from '@vitest/runner/utils' +export type { ErrorWithDiff, ParsedStack } from '@vitest/utils' export type Awaitable = T | PromiseLike export type Nullable = T | null | undefined diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ae58878a5e6c..0cb6dcb68d31 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -970,6 +970,9 @@ importers: specifier: ^4.3.7 version: 4.3.7 devDependencies: + '@vitest/runner': + specifier: workspace:* + version: link:../runner picocolors: specifier: ^1.0.0 version: 1.0.0 @@ -1596,6 +1599,15 @@ importers: specifier: workspace:* version: link:../../packages/vitest + test/failing: + devDependencies: + '@vitest/runner': + specifier: workspace:* + version: link:../../packages/runner + vitest: + specifier: workspace:* + version: link:../../packages/vitest + test/fails: devDependencies: '@vitest/coverage-istanbul': @@ -15482,7 +15494,7 @@ packages: dependencies: foreground-child: 3.1.1 jackspeak: 2.1.1 - minimatch: 9.0.0 + minimatch: 9.0.1 minipass: 5.0.0 path-scurry: 1.7.0 dev: true @@ -17659,7 +17671,7 @@ packages: cssom: 0.5.0 cssstyle: 2.3.0 data-urls: 3.0.2 - decimal.js: 10.4.2 + decimal.js: 10.4.3 domexception: 4.0.0 escodegen: 2.0.0 form-data: 4.0.0 @@ -17667,8 +17679,8 @@ packages: http-proxy-agent: 5.0.0 https-proxy-agent: 5.0.1 is-potential-custom-element-name: 1.0.1 - nwsapi: 2.2.2 - parse5: 7.1.1 + nwsapi: 2.2.4 + parse5: 7.1.2 saxes: 6.0.0 symbol-tree: 3.2.4 tough-cookie: 4.1.2 diff --git a/test/core/test/error.test.ts b/test/core/test/error.test.ts index eb186ea60a8c..02c39af569d2 100644 --- a/test/core/test/error.test.ts +++ b/test/core/test/error.test.ts @@ -1,4 +1,4 @@ -import { processError } from '@vitest/runner/utils' +import { processError } from '@vitest/utils/error' import { expect, test } from 'vitest' test('Can correctly process error where actual and expected contains non writable properties', () => { diff --git a/test/core/test/expect.test.ts b/test/core/test/expect.test.ts new file mode 100644 index 000000000000..ce12e36c344d --- /dev/null +++ b/test/core/test/expect.test.ts @@ -0,0 +1,43 @@ +import { getCurrentTest } from '@vitest/runner' +import { describe, expect, expectTypeOf, test } from 'vitest' + +describe('expect.soft', () => { + test('types', () => { + expectTypeOf(expect.soft).toEqualTypeOf(expect) + expectTypeOf(expect.soft(7)).toEqualTypeOf(expect(7)) + expectTypeOf(expect.soft(5)).toHaveProperty('toBe') + expectTypeOf(expect.soft(7)).not.toHaveProperty('toCustom') + }) + + test('return value', () => { + expect(expect.soft('test')).toHaveProperty('toBe') + expect(expect.soft('test')).toHaveProperty('toEqual') + }) + + test('with extend', () => { + expect.extend({ + toBeFoo(received) { + const { isNot } = this + return { + // do not alter your "pass" based on isNot. Vitest does it for you + pass: received === 'foo', + message: () => `${received} is${isNot ? ' not' : ''} foo`, + } + }, + }) + expect(expect.soft('test')).toHaveProperty('toBeFoo') + }) + + test('should have multiple error', () => { + expect.soft(1).toBe(2) + expect.soft(2).toBe(3) + getCurrentTest()!.result!.state = 'run' + expect(getCurrentTest()?.result?.errors).toHaveLength(2) + }) + + test.fails('should be a failure', () => { + expect.soft('test1').toBe('test res') + expect.soft('test2').toBe('test res') + expect.soft('test3').toBe('test res') + }) +}) diff --git a/test/core/test/replace-matcher.test.ts b/test/core/test/replace-matcher.test.ts index d6e21c4f6c52..3c77007650b0 100644 --- a/test/core/test/replace-matcher.test.ts +++ b/test/core/test/replace-matcher.test.ts @@ -1,5 +1,5 @@ +import { replaceAsymmetricMatcher } from '@vitest/utils/error' import { describe, expect, it } from 'vitest' -import { replaceAsymmetricMatcher } from '@vitest/runner/utils' describe('replace asymmetric matcher', () => { const expectReplaceAsymmetricMatcher = (actual: any, expected: any) => { diff --git a/test/core/test/serialize.test.ts b/test/core/test/serialize.test.ts index c54b5af84f19..2b29aa728b4f 100644 --- a/test/core/test/serialize.test.ts +++ b/test/core/test/serialize.test.ts @@ -1,7 +1,7 @@ // @vitest-environment jsdom +import { serializeError } from '@vitest/utils/error' import { describe, expect, it } from 'vitest' -import { serializeError } from '@vitest/runner/utils' describe('error serialize', () => { it('works', () => { diff --git a/test/core/vitest.config.ts b/test/core/vitest.config.ts index 44f1710f9261..b869e9d40aea 100644 --- a/test/core/vitest.config.ts +++ b/test/core/vitest.config.ts @@ -1,5 +1,5 @@ import { basename, dirname, join, resolve } from 'pathe' -import { defineConfig } from 'vitest/config' +import { defaultExclude, defineConfig } from 'vitest/config' export default defineConfig({ plugins: [ @@ -41,6 +41,7 @@ export default defineConfig({ }, test: { name: 'core', + exclude: ['**/fixtures/**', ...defaultExclude], slowTestThreshold: 1000, testTimeout: 2000, setupFiles: [ diff --git a/test/failing/fixtures/expects/soft.test.ts b/test/failing/fixtures/expects/soft.test.ts new file mode 100644 index 000000000000..d7bf8c48744e --- /dev/null +++ b/test/failing/fixtures/expects/soft.test.ts @@ -0,0 +1,85 @@ +import { expect, test } from 'vitest' + +interface CustomMatchers { + toBeDividedBy(divisor: number): R +} + +declare module 'vitest' { + interface Assertion extends CustomMatchers {} +} + +expect.extend({ + toBeDividedBy(received, divisor) { + const pass = received % divisor === 0 + if (pass) { + return { + message: () => + `expected ${received} not to be divisible by ${divisor}`, + pass: true, + } + } + else { + return { + message: () => + `expected ${received} to be divisible by ${divisor}`, + pass: false, + } + } + }, +}) + +test('basic', () => { + expect.soft(1).toBe(2) + expect.soft(2).toBe(3) +}) + +test('promise', async () => { + await expect.soft( + new Promise((resolve) => { + setTimeout(() => { + resolve(1) + }) + }), + ).resolves.toBe(2) + await expect.soft( + new Promise((resolve) => { + setTimeout(() => { + resolve(2) + }) + }), + ).resolves.toBe(3) +}) + +test('with expect', () => { + expect.soft(1).toEqual(2) + expect(10).toEqual(20) + expect.soft(2).toEqual(3) +}) + +test('with expect.extend', () => { + expect.soft(1).toEqual(2) + expect.soft(3).toBeDividedBy(4) + expect(5).toEqual(6) +}) + +test('passed', () => { + expect.soft(1).toEqual(1) + expect(10).toEqual(10) + expect.soft(2).toEqual(2) +}) + +let num = 0 +test('retry will passed', () => { + expect.soft(num += 1).toBe(3) + expect.soft(num += 1).toBe(4) +}, { + retry: 2, +}) + +num = 0 +test('retry will failed', () => { + expect.soft(num += 1).toBe(4) + expect.soft(num += 1).toBe(5) +}, { + retry: 2, +}) diff --git a/test/failing/fixtures/vite.config.ts b/test/failing/fixtures/vite.config.ts new file mode 100644 index 000000000000..6d0258279ded --- /dev/null +++ b/test/failing/fixtures/vite.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vite' + +export default defineConfig({ + test: { + threads: false, + isolate: false, + }, +}) diff --git a/test/failing/package.json b/test/failing/package.json new file mode 100644 index 000000000000..6bba674dcb7e --- /dev/null +++ b/test/failing/package.json @@ -0,0 +1,12 @@ +{ + "name": "@vitest/test-failing", + "private": true, + "scripts": { + "test": "vitest", + "coverage": "vitest run --coverage" + }, + "devDependencies": { + "@vitest/runner": "workspace:*", + "vitest": "workspace:*" + } +} diff --git a/test/failing/test/expect.test.ts b/test/failing/test/expect.test.ts new file mode 100644 index 000000000000..852684adf778 --- /dev/null +++ b/test/failing/test/expect.test.ts @@ -0,0 +1,54 @@ +import { resolve } from 'node:path' +import type { UserConfig } from 'vitest' +import { describe, expect, test } from 'vitest' +import { getCurrentTest } from '@vitest/runner' +import { runVitest } from '../../test-utils' + +describe('expect.soft', () => { + const run = (config?: UserConfig) => runVitest({ root: resolve('./fixtures'), include: ['expects/soft.test.ts'], setupFiles: [], testNamePattern: getCurrentTest()?.name, testTimeout: 4000, ...config }, ['soft']) + + test('basic', async () => { + const { stderr } = await run() + expect.soft(stderr).toContain('AssertionError: expected 1 to be 2') + expect.soft(stderr).toContain('AssertionError: expected 2 to be 3') + }) + + test('promise', async () => { + const { stderr } = await run() + expect.soft(stderr).toContain('AssertionError: expected 2 to be 3') + expect.soft(stderr).toContain('AssertionError: expected 1 to be 2') + }) + + test('with expect', async () => { + const { stderr } = await run() + expect.soft(stderr).toContain('AssertionError: expected 1 to deeply equal 2') + expect.soft(stderr).toContain('AssertionError: expected 10 to deeply equal 20') + expect.soft(stderr).not.toContain('AssertionError: expected 2 to deeply equal 3') + }) + + test('with expect.extend', async () => { + const { stderr } = await run() + expect.soft(stderr).toContain('AssertionError: expected 1 to deeply equal 2') + expect.soft(stderr).toContain('Error: expected 3 to be divisible by 4') + expect.soft(stderr).toContain('AssertionError: expected 5 to deeply equal 6') + }) + + test('passed', async () => { + const { stdout } = await run() + expect.soft(stdout).toContain('soft.test.ts > passed') + }) + + test('retry will passed', async () => { + const { stdout } = await run() + expect.soft(stdout).toContain('soft.test.ts > retry will passed') + }) + + test('retry will failed', async () => { + const { stderr } = await run() + expect.soft(stderr).toContain('AssertionError: expected 1 to be 4') + expect.soft(stderr).toContain('AssertionError: expected 2 to be 5') + expect.soft(stderr).toContain('AssertionError: expected 3 to be 4') + expect.soft(stderr).toContain('AssertionError: expected 4 to be 5') + expect.soft(stderr).toContain('4/4') + }) +}, 4000) diff --git a/test/failing/vite.config.ts b/test/failing/vite.config.ts new file mode 100644 index 000000000000..a36f56b70ba0 --- /dev/null +++ b/test/failing/vite.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vite' + +export default defineConfig({ + test: { + include: ['test/*.test.ts'], + chaiConfig: { + truncateThreshold: 9999, + }, + }, +}) diff --git a/test/fails/fixtures/expect-soft.test.ts b/test/fails/fixtures/expect-soft.test.ts new file mode 100644 index 000000000000..2691ad966be0 --- /dev/null +++ b/test/fails/fixtures/expect-soft.test.ts @@ -0,0 +1,3 @@ +import { expect } from 'vitest' + +expect.soft(1 + 1).toEqual(3) diff --git a/test/fails/test/__snapshots__/runner.test.ts.snap b/test/fails/test/__snapshots__/runner.test.ts.snap index ac8c44666374..0dca905eb9e7 100644 --- a/test/fails/test/__snapshots__/runner.test.ts.snap +++ b/test/fails/test/__snapshots__/runner.test.ts.snap @@ -8,6 +8,8 @@ exports[`should fail empty.test.ts > empty.test.ts 1`] = `"Error: No test suite exports[`should fail expect.test.ts > expect.test.ts 1`] = `"AssertionError: expected 2 to deeply equal 3"`; +exports[`should fail expect-soft.test.ts > expect-soft.test.ts 1`] = `"Error: expect.soft() can only be used inside a test"`; + exports[`should fail hook-timeout.test.ts > hook-timeout.test.ts 1`] = `"Error: Hook timed out in 10ms."`; exports[`should fail hooks-called.test.ts > hooks-called.test.ts 1`] = `