feat(expect): support expect.soft (#3507)

packages/expect/package.json

@@ -39,6 +39,7 @@
     "chai": "^4.3.7"
   },
   "devDependencies": {
+    "@vitest/runner": "workspace:*",
     "picocolors": "^1.0.0"
   }
 }

packages/expect/rollup.config.js

@@ -9,6 +9,7 @@ const external = [
   ...Object.keys(pkg.dependencies || {}),
   ...Object.keys(pkg.peerDependencies || {}),
   '@vitest/utils/diff',
+  '@vitest/utils/error',
 ]
 
 const plugins = [

packages/expect/src/jest-expect.ts

@@ -3,21 +3,23 @@ import { assertTypes, getColors } from '@vitest/utils'
 import type { Constructable } from '@vitest/utils'
 import type { EnhancedSpy } from '@vitest/spy'
 import { isMockFunction } from '@vitest/spy'
+import type { Test } from '@vitest/runner'
 import type { Assertion, ChaiPlugin } from './types'
 import { arrayBufferEquality, generateToBeMessage, iterableEquality, equals as jestEquals, sparseArrayEquality, subsetEquality, typeEquality } from './jest-utils'
 import type { AsymmetricMatcher } from './jest-asymmetric-matchers'
 import { diff, stringify } from './jest-matcher-utils'
 import { JEST_MATCHERS_OBJECT } from './constants'
-import { recordAsyncExpect } from './utils'
+import { recordAsyncExpect, wrapSoft } from './utils'
 
 // Jest Expect Compact
 export const JestChaiExpect: ChaiPlugin = (chai, utils) => {
   const c = () => getColors()
 
   function def(name: keyof Assertion | (keyof Assertion)[], fn: ((this: Chai.AssertionStatic & Assertion, ...args: any[]) => any)) {
     const addMethod = (n: keyof Assertion) => {
-      utils.addMethod(chai.Assertion.prototype, n, fn)
-      utils.addMethod((globalThis as any)[JEST_MATCHERS_OBJECT].matchers, n, fn)
+      const softWrapper = wrapSoft(utils, fn)
+      utils.addMethod(chai.Assertion.prototype, n, softWrapper)
+      utils.addMethod((globalThis as any)[JEST_MATCHERS_OBJECT].matchers, n, softWrapper)
     }
 
     if (Array.isArray(name))
@@ -636,7 +638,7 @@ export const JestChaiExpect: ChaiPlugin = (chai, utils) => {
   utils.addProperty(chai.Assertion.prototype, 'resolves', function __VITEST_RESOLVES__(this: any) {
     utils.flag(this, 'promise', 'resolves')
     utils.flag(this, 'error', new Error('resolves'))
-    const test = utils.flag(this, 'vitest-test')
+    const test: Test = utils.flag(this, 'vitest-test')
     const obj = utils.flag(this, 'object')
 
     if (typeof obj?.then !== 'function')
@@ -671,7 +673,7 @@ export const JestChaiExpect: ChaiPlugin = (chai, utils) => {
   utils.addProperty(chai.Assertion.prototype, 'rejects', function __VITEST_REJECTS__(this: any) {
     utils.flag(this, 'promise', 'rejects')
     utils.flag(this, 'error', new Error('rejects'))
-    const test = utils.flag(this, 'vitest-test')
+    const test: Test = utils.flag(this, 'vitest-test')
     const obj = utils.flag(this, 'object')
     const wrapper = typeof obj === 'function' ? obj() : obj // for jest compat
 

packages/expect/src/jest-extend.ts

@@ -17,6 +17,7 @@ import {
   iterableEquality,
   subsetEquality,
 } from './jest-utils'
+import { wrapSoft } from './utils'
 
 function getMatcherState(assertion: Chai.AssertionStatic & Chai.Assertion, expect: ExpectStatic) {
   const obj = assertion._obj
@@ -75,8 +76,9 @@ function JestExtendPlugin(expect: ExpectStatic, matchers: MatchersObject): ChaiP
           throw new JestExtendError(message(), actual, expected)
       }
 
-      utils.addMethod((globalThis as any)[JEST_MATCHERS_OBJECT].matchers, expectAssertionName, expectWrapper)
-      utils.addMethod(c.Assertion.prototype, expectAssertionName, expectWrapper)
+      const softWrapper = wrapSoft(utils, expectWrapper)
+      utils.addMethod((globalThis as any)[JEST_MATCHERS_OBJECT].matchers, expectAssertionName, softWrapper)
+      utils.addMethod(c.Assertion.prototype, expectAssertionName, softWrapper)
 
       class CustomMatcher extends AsymmetricMatcher<[unknown, ...unknown[]]> {
         constructor(inverse = false, ...sample: [unknown, ...unknown[]]) {

packages/expect/src/types.ts

@@ -79,6 +79,7 @@ export interface MatcherState {
     iterableEquality: Tester
     subsetEquality: Tester
   }
+  soft?: boolean
 }
 
 export interface SyncExpectationResult {
@@ -100,7 +101,7 @@ export type MatchersObject<T extends MatcherState = MatcherState> = Record<strin
 
 export interface ExpectStatic extends Chai.ExpectStatic, AsymmetricMatchersContaining {
   <T>(actual: T, message?: string): Assertion<T>
-
+  soft<T>(actual: T, message?: string): Assertion<T>
   extend(expects: MatchersObject): void
   assertions(expected: number): void
   hasAssertions(): void

packages/expect/src/utils.ts

@@ -1,3 +1,9 @@
+import { processError } from '@vitest/utils/error'
+import type { Test } from '@vitest/runner/types'
+import { GLOBAL_EXPECT } from './constants'
+import { getState } from './state'
+import type { Assertion, MatcherState } from './types'
+
 export function recordAsyncExpect(test: any, promise: Promise<any> | PromiseLike<any>) {
   // record promise for test, that resolves before test ends
   if (test && promise instanceof Promise) {
@@ -16,3 +22,30 @@ export function recordAsyncExpect(test: any, promise: Promise<any> | PromiseLike
 
   return promise
 }
+
+export function wrapSoft(utils: Chai.ChaiUtils, fn: (this: Chai.AssertionStatic & Assertion, ...args: any[]) => void) {
+  return function (this: Chai.AssertionStatic & Assertion, ...args: any[]) {
+    const test: Test = utils.flag(this, 'vitest-test')
+
+    // @ts-expect-error local is untyped
+    const state: MatcherState = test?.context._local
+      ? test.context.expect.getState()
+      : getState((globalThis as any)[GLOBAL_EXPECT])
+
+    if (!state.soft)
+      return fn.apply(this, args)
+
+    if (!test)
+      throw new Error('expect.soft() can only be used inside a test')
+
+    try {
+      return fn.apply(this, args)
+    }
+    catch (err) {
+      test.result ||= { state: 'fail' }
+      test.result.state = 'fail'
+      test.result.errors ||= []
+      test.result.errors.push(processError(err))
+    }
+  }
+}

packages/runner/rollup.config.js

@@ -9,7 +9,7 @@ const external = [
   ...builtinModules,
   ...Object.keys(pkg.dependencies || {}),
   ...Object.keys(pkg.peerDependencies || {}),
-  '@vitest/utils/diff',
+  '@vitest/utils/error',
 ]
 
 const entries = {

packages/runner/src/collect.ts

@@ -1,10 +1,10 @@
 import { relative } from 'pathe'
+import { processError } from '@vitest/utils/error'
 import type { File } from './types'
 import type { VitestRunner } from './types/runner'
 import { calculateSuiteHash, generateHash, interpretTaskModes, someTasksAreOnly } from './utils/collect'
 import { clearCollectorContext, getDefaultSuite } from './suite'
 import { getHooks, setHooks } from './map'
-import { processError } from './utils/error'
 import { collectorContext } from './context'
 import { runSetupFiles } from './setup'
 

packages/runner/src/run.ts

@@ -1,11 +1,11 @@
 import limit from 'p-limit'
 import { getSafeTimers, shuffle } from '@vitest/utils'
+import { processError } from '@vitest/utils/error'
 import type { VitestRunner } from './types/runner'
 import type { File, HookCleanupCallback, HookListener, SequenceHooks, Suite, SuiteHooks, Task, TaskMeta, TaskResult, TaskResultPack, TaskState, Test } from './types'
 import { partitionSuiteChildren } from './utils/suite'
 import { getFn, getHooks } from './map'
 import { collectTests } from './collect'
-import { processError } from './utils/error'
 import { setCurrentTest } from './test-state'
 import { hasFailed, hasTests } from './utils/tasks'
 
@@ -156,7 +156,6 @@ export async function runTest(test: Test, runner: VitestRunner) {
             throw new Error('Test function is not found. Did you add it using `setFn`?')
           await fn()
         }
-
         // some async expect will be added to this array, in case user forget to await theme
         if (test.promises) {
           const result = await Promise.allSettled(test.promises)
@@ -167,10 +166,12 @@ export async function runTest(test: Test, runner: VitestRunner) {
 
         await runner.onAfterTryTest?.(test, { retry: retryCount, repeats: repeatCount })
 
-        if (!test.repeats)
-          test.result.state = 'pass'
-        else if (test.repeats && retry === retryCount)
-          test.result.state = 'pass'
+        if (test.result.state !== 'fail') {
+          if (!test.repeats)
+            test.result.state = 'pass'
+          else if (test.repeats && retry === retryCount)
+            test.result.state = 'pass'
+        }
       }
       catch (e) {
         failTask(test.result, e)
@@ -186,6 +187,12 @@ export async function runTest(test: Test, runner: VitestRunner) {
 
       if (test.result.state === 'pass')
         break
+
+      if (retryCount < retry - 1) {
+        // reset state when retry test
+        test.result.state = 'run'
+      }
+
       // update retry info
       updateTask(test, runner)
     }

packages/runner/src/types/tasks.ts

@@ -1,6 +1,5 @@
-import type { Awaitable } from '@vitest/utils'
+import type { Awaitable, ErrorWithDiff } from '@vitest/utils'
 import type { ChainableFunction } from '../utils/chain'
-import type { ErrorWithDiff } from '../utils/error'
 
 export type RunMode = 'run' | 'skip' | 'only' | 'todo'
 export type TaskState = RunMode | 'pass' | 'fail'

packages/runner/src/utils/collect.ts

@@ -1,5 +1,5 @@
+import { processError } from '@vitest/utils/error'
 import type { Suite, TaskBase } from '../types'
-import { processError } from './error'
 
 /**
  * If any tasks been marked as `only`, mark all other tasks as `skip`.