✨ No timeout for beforeEach or afterEach (#3464)

Timeout must only focus on the code under tests, in other words the predicate. It should not be triggered because of a beforeEach being too long, or a afterEach being too long.
dubzzz · Dec 10, 2022 · b330dd3 · b330dd3
1 parent 4c153e6
commit b330dd3
Show file tree

Hide file tree

Showing 16 changed files with 633 additions and 101 deletions.
diff --git a/.yarn/versions/d036f217.yml b/.yarn/versions/d036f217.yml
@@ -0,0 +1,7 @@
+releases:
+  fast-check: minor
+
+declined:
+  - "@fast-check/ava"
+  - "@fast-check/jest"
+  - "@fast-check/worker"
diff --git a/packages/fast-check/src/check/property/AsyncProperty.generic.ts b/packages/fast-check/src/check/property/AsyncProperty.generic.ts
@@ -101,8 +101,18 @@ export class AsyncProperty<Ts> implements IAsyncPropertyWithHooks<Ts> {
     return this.arb.shrink(value.value_, safeContext).map(noUndefinedAsContext);
   }
 
-  async run(v: Ts): Promise<PreconditionFailure | PropertyFailure | null> {
+  async runBeforeEach(): Promise<void> {
     await this.beforeEachHook();
+  }
+
+  async runAfterEach(): Promise<void> {
+    await this.afterEachHook();
+  }
+
+  async run(v: Ts, dontRunHook?: boolean): Promise<PreconditionFailure | PropertyFailure | null> {
+    if (!dontRunHook) {
+      await this.beforeEachHook();
+    }
     try {
       const output = await this.predicate(v);
       return output == null || output === true
@@ -120,7 +130,9 @@ export class AsyncProperty<Ts> implements IAsyncPropertyWithHooks<Ts> {
       }
       return { error: err, errorMessage: String(err) };
     } finally {
-      await this.afterEachHook();
+      if (!dontRunHook) {
+        await this.afterEachHook();
+      }
     }
   }
 

diff --git a/packages/fast-check/src/check/property/IRawProperty.ts b/packages/fast-check/src/check/property/IRawProperty.ts
@@ -66,13 +66,27 @@ export interface IRawProperty<Ts, IsAsync extends boolean = boolean> {
   /**
    * Check the predicate for v
    * @param v - Value of which we want to check the predicate
+   * @param dontRunHook - Do not run beforeEach and afterEach hooks within run
    * @remarks Since 0.0.7
    */
   run(
-    v: Ts
+    v: Ts,
+    dontRunHook?: boolean
   ):
     | (IsAsync extends true ? Promise<PreconditionFailure | PropertyFailure | null> : never)
     | (IsAsync extends false ? PreconditionFailure | PropertyFailure | null : never);
+
+  /**
+   * Run before each hook
+   * @remarks Since 3.4.0
+   */
+  runBeforeEach?: () => (IsAsync extends true ? Promise<void> : never) | (IsAsync extends false ? void : never);
+
+  /**
+   * Run after each hook
+   * @remarks Since 3.4.0
+   */
+  runAfterEach?: () => (IsAsync extends true ? Promise<void> : never) | (IsAsync extends false ? void : never);
 }
 
 /**

diff --git a/packages/fast-check/src/check/property/IgnoreEqualValuesProperty.ts b/packages/fast-check/src/check/property/IgnoreEqualValuesProperty.ts
@@ -39,9 +39,18 @@ function fromCachedUnsafe<Ts, IsAsync extends boolean>(
 
 /** @internal */
 export class IgnoreEqualValuesProperty<Ts, IsAsync extends boolean> implements IRawProperty<Ts, IsAsync> {
+  runBeforeEach?: () => (IsAsync extends true ? Promise<void> : never) | (IsAsync extends false ? void : never);
+  runAfterEach?: () => (IsAsync extends true ? Promise<void> : never) | (IsAsync extends false ? void : never);
   private coveredCases: Map<string, ReturnType<IRawProperty<Ts, IsAsync>['run']>> = new Map();
 
-  constructor(readonly property: IRawProperty<Ts, IsAsync>, readonly skipRuns: boolean) {}
+  constructor(readonly property: IRawProperty<Ts, IsAsync>, readonly skipRuns: boolean) {
+    if (this.property.runBeforeEach !== undefined && this.property.runAfterEach !== undefined) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.runBeforeEach = () => this.property.runBeforeEach!();
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.runAfterEach = () => this.property.runAfterEach!();
+    }
+  }
 
   isAsync(): IsAsync {
     return this.property.isAsync();
@@ -55,7 +64,7 @@ export class IgnoreEqualValuesProperty<Ts, IsAsync extends boolean> implements I
     return this.property.shrink(value);
   }
 
-  run(v: Ts): ReturnType<IRawProperty<Ts, IsAsync>['run']> {
+  run(v: Ts, dontRunHook: boolean): ReturnType<IRawProperty<Ts, IsAsync>['run']> {
     const stringifiedValue = stringify(v);
     if (this.coveredCases.has(stringifiedValue)) {
       const lastOutput = this.coveredCases.get(stringifiedValue) as ReturnType<IRawProperty<Ts, IsAsync>['run']>;
@@ -64,7 +73,7 @@ export class IgnoreEqualValuesProperty<Ts, IsAsync extends boolean> implements I
       }
       return fromCachedUnsafe(lastOutput, this.property.isAsync());
     }
-    const out = this.property.run(v);
+    const out = this.property.run(v, dontRunHook);
     this.coveredCases.set(stringifiedValue, out);
     return out;
   }

diff --git a/packages/fast-check/src/check/property/Property.generic.ts b/packages/fast-check/src/check/property/Property.generic.ts
@@ -117,8 +117,18 @@ export class Property<Ts> implements IProperty<Ts>, IPropertyWithHooks<Ts> {
     return this.arb.shrink(value.value_, safeContext).map(noUndefinedAsContext);
   }
 
-  run(v: Ts): PreconditionFailure | PropertyFailure | null {
+  runBeforeEach(): void {
     this.beforeEachHook();
+  }
+
+  runAfterEach(): void {
+    this.afterEachHook();
+  }
+
+  run(v: Ts, dontRunHook?: boolean): PreconditionFailure | PropertyFailure | null {
+    if (!dontRunHook) {
+      this.beforeEachHook();
+    }
     try {
       const output = this.predicate(v);
       return output == null || output === true
@@ -136,7 +146,9 @@ export class Property<Ts> implements IProperty<Ts>, IPropertyWithHooks<Ts> {
       }
       return { error: err, errorMessage: String(err) };
     } finally {
-      this.afterEachHook();
+      if (!dontRunHook) {
+        this.afterEachHook();
+      }
     }
   }
 

diff --git a/packages/fast-check/src/check/property/SkipAfterProperty.ts b/packages/fast-check/src/check/property/SkipAfterProperty.ts
@@ -6,14 +6,23 @@ import { IRawProperty } from './IRawProperty';
 
 /** @internal */
 export class SkipAfterProperty<Ts, IsAsync extends boolean> implements IRawProperty<Ts, IsAsync> {
+  runBeforeEach?: () => (IsAsync extends true ? Promise<void> : never) | (IsAsync extends false ? void : never);
+  runAfterEach?: () => (IsAsync extends true ? Promise<void> : never) | (IsAsync extends false ? void : never);
   private skipAfterTime: number;
+
   constructor(
     readonly property: IRawProperty<Ts, IsAsync>,
     readonly getTime: () => number,
     timeLimit: number,
     readonly interruptExecution: boolean
   ) {
     this.skipAfterTime = this.getTime() + timeLimit;
+    if (this.property.runBeforeEach !== undefined && this.property.runAfterEach !== undefined) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.runBeforeEach = () => this.property.runBeforeEach!();
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.runAfterEach = () => this.property.runAfterEach!();
+    }
   }
 
   isAsync(): IsAsync {
@@ -28,7 +37,7 @@ export class SkipAfterProperty<Ts, IsAsync extends boolean> implements IRawPrope
     return this.property.shrink(value);
   }
 
-  run(v: Ts): ReturnType<IRawProperty<Ts, IsAsync>['run']> {
+  run(v: Ts, dontRunHook: boolean): ReturnType<IRawProperty<Ts, IsAsync>['run']> {
     if (this.getTime() >= this.skipAfterTime) {
       const preconditionFailure = new PreconditionFailure(this.interruptExecution);
       if (this.isAsync()) {
@@ -37,6 +46,6 @@ export class SkipAfterProperty<Ts, IsAsync extends boolean> implements IRawPrope
         return preconditionFailure as any; // !IsAsync => PreconditionFailure | string | null
       }
     }
-    return this.property.run(v);
+    return this.property.run(v, dontRunHook);
   }
 }
diff --git a/packages/fast-check/src/check/property/TimeoutProperty.ts b/packages/fast-check/src/check/property/TimeoutProperty.ts
@@ -26,7 +26,17 @@ const timeoutAfter = (timeMs: number) => {
 
 /** @internal */
 export class TimeoutProperty<Ts> implements IRawProperty<Ts, true> {
-  constructor(readonly property: IRawProperty<Ts>, readonly timeMs: number) {}
+  runBeforeEach?: () => Promise<void>;
+  runAfterEach?: () => Promise<void>;
+
+  constructor(readonly property: IRawProperty<Ts>, readonly timeMs: number) {
+    if (this.property.runBeforeEach !== undefined && this.property.runAfterEach !== undefined) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.runBeforeEach = () => Promise.resolve(this.property.runBeforeEach!());
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.runAfterEach = () => Promise.resolve(this.property.runAfterEach!());
+    }
+  }
 
   isAsync(): true {
     return true;
@@ -40,9 +50,9 @@ export class TimeoutProperty<Ts> implements IRawProperty<Ts, true> {
     return this.property.shrink(value);
   }
 
-  async run(v: Ts): Promise<PreconditionFailure | PropertyFailure | null> {
+  async run(v: Ts, dontRunHook: boolean): Promise<PreconditionFailure | PropertyFailure | null> {
     const t = timeoutAfter(this.timeMs);
-    const propRun = Promise.race([this.property.run(v), t.promise]);
+    const propRun = Promise.race([this.property.run(v, dontRunHook), t.promise]);
     propRun.then(t.clear, t.clear); // always clear timeout handle - catch should never occur
     return propRun;
   }

diff --git a/packages/fast-check/src/check/property/UnbiasedProperty.ts b/packages/fast-check/src/check/property/UnbiasedProperty.ts
@@ -5,7 +5,17 @@ import { IRawProperty } from './IRawProperty';
 
 /** @internal */
 export class UnbiasedProperty<Ts, IsAsync extends boolean> implements IRawProperty<Ts, IsAsync> {
-  constructor(readonly property: IRawProperty<Ts, IsAsync>) {}
+  runBeforeEach?: () => (IsAsync extends true ? Promise<void> : never) | (IsAsync extends false ? void : never);
+  runAfterEach?: () => (IsAsync extends true ? Promise<void> : never) | (IsAsync extends false ? void : never);
+
+  constructor(readonly property: IRawProperty<Ts, IsAsync>) {
+    if (this.property.runBeforeEach !== undefined && this.property.runAfterEach !== undefined) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.runBeforeEach = () => this.property.runBeforeEach!();
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      this.runAfterEach = () => this.property.runAfterEach!();
+    }
+  }
 
   isAsync(): IsAsync {
     return this.property.isAsync();
@@ -19,7 +29,7 @@ export class UnbiasedProperty<Ts, IsAsync extends boolean> implements IRawProper
     return this.property.shrink(value);
   }
 
-  run(v: Ts): ReturnType<IRawProperty<Ts, IsAsync>['run']> {
-    return this.property.run(v);
+  run(v: Ts, dontRunHook: boolean): ReturnType<IRawProperty<Ts, IsAsync>['run']> {
+    return this.property.run(v, dontRunHook);
   }
 }
diff --git a/packages/fast-check/src/check/runner/Runner.ts b/packages/fast-check/src/check/runner/Runner.ts
@@ -27,9 +27,18 @@ function runIt<Ts>(
   verbose: VerbosityLevel,
   interruptedAsFailure: boolean
 ): RunExecution<Ts> {
+  const isModernProperty = property.runBeforeEach !== undefined && property.runAfterEach !== undefined;
   const runner = new RunnerIterator(sourceValues, shrink, verbose, interruptedAsFailure);
   for (const v of runner) {
-    const out = property.run(v) as PreconditionFailure | PropertyFailure | null;
+    if (isModernProperty) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      property.runBeforeEach!();
+    }
+    const out = property.run(v, isModernProperty) as PreconditionFailure | PropertyFailure | null;
+    if (isModernProperty) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      property.runAfterEach!();
+    }
     runner.handleResult(out);
   }
   return runner.runExecution;
@@ -43,9 +52,18 @@ async function asyncRunIt<Ts>(
   verbose: VerbosityLevel,
   interruptedAsFailure: boolean
 ): Promise<RunExecution<Ts>> {
+  const isModernProperty = property.runBeforeEach !== undefined && property.runAfterEach !== undefined;
   const runner = new RunnerIterator(sourceValues, shrink, verbose, interruptedAsFailure);
   for (const v of runner) {
-    const out = await property.run(v);
+    if (isModernProperty) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      await property.runBeforeEach!();
+    }
+    const out = await property.run(v, isModernProperty);
+    if (isModernProperty) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      await property.runAfterEach!();
+    }
     runner.handleResult(out);
   }
   return runner.runExecution;

diff --git a/packages/fast-check/test/e2e/Timeout.spec.ts b/packages/fast-check/test/e2e/Timeout.spec.ts
@@ -0,0 +1,28 @@
+import * as fc from '../../src/fast-check';
+import { seed } from './seed';
+
+describe(`Timeout (seed: ${seed})`, () => {
+  it('should always run beforeEach and afterEach even in case of timeout', async () => {
+    let numRuns = 0;
+    const beforeEach = jest.fn().mockResolvedValue(undefined);
+    const afterEach = jest.fn().mockResolvedValue(undefined);
+    const out = await fc.check(
+      fc
+        .asyncProperty(fc.integer().noShrink(), async (_x) => {
+          ++numRuns;
+          await new Promise(() => {}); // never ending promise
+        })
+        .beforeEach(beforeEach)
+        .afterEach(afterEach),
+      { timeout: 0 }
+    );
+    expect(out.failed).toBe(true);
+    expect(out.interrupted).toBe(false);
+    expect(out.numRuns).toBe(1); // only once, it timeouts on first run and then shrink (no-shrink here)
+    expect(out.numShrinks).toBe(0);
+    expect(out.numSkips).toBe(0);
+    expect(numRuns).toBe(1);
+    expect(beforeEach).toHaveBeenCalledTimes(1);
+    expect(afterEach).toHaveBeenCalledTimes(1);
+  });
+});