feat: add workerIdleMemoryLimit to support worker recycling in the event of node >16.11.0 memory leaks

CHANGELOG.md

@@ -5,6 +5,7 @@
 - `[jest-config]` [**BREAKING**] Make `snapshotFormat` default to `escapeString: false` and `printBasicPrototype: false` ([#13036](https://github.com/facebook/jest/pull/13036))
 - `[jest-environment-jsdom]` [**BREAKING**] Upgrade to `jsdom@20` ([#13037](https://github.com/facebook/jest/pull/13037))
 - `[pretty-format]` [**BREAKING**] Remove `ConvertAnsi` plugin in favour of `jest-serializer-ansi-escapes` ([#13040](https://github.com/facebook/jest/pull/13040))
+- `[jest-worker]` Adds `workerIdleMemoryLimit` option which is used as a check for worker memory leaks >= Node 16.11.0 and recycles child workers as required. ([#13056](https://github.com/facebook/jest/pull/13056))
 
 ### Fixes
 

docs/Configuration.md

@@ -2267,3 +2267,21 @@ This option allows comments in `package.json`. Include the comment text as the v
   }
 }
 ```
+
+### `workerIdleMemoryLimit` \[number]
+
+Specifies the memory limit for workers before they are recycled and is primarily a work-around for [this issue](https://github.com/facebook/jest/issues/11956);
+
+After the worker has executed a test the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. The limit can be specified in 2 ways
+
+* < 1 - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory
+* > 1 - Assumed to be a fixed byte value
+
+```js tab
+/** @type {import('jest').Config} */
+const config = {
+  workerIdleMemoryLimit: 0.2,
+};
+
+module.exports = config;
+```

packages/jest-runner/src/index.ts

@@ -107,6 +107,7 @@ export default class TestRunner extends EmittingTestRunner {
     const worker = new Worker(require.resolve('./testWorker'), {
       exposedMethods: ['worker'],
       forkOptions: {serialization: 'json', stdio: 'pipe'},
+      idleMemoryLimit: this._globalConfig.workerIdleMemoryLimit,
       maxRetries: 3,
       numWorkers: this._globalConfig.maxWorkers,
       setupArgs: [{serializableResolvers: Array.from(resolvers.values())}],

packages/jest-types/src/Config.ts

@@ -327,6 +327,7 @@ export type InitialOptions = Partial<{
   watchAll: boolean;
   watchman: boolean;
   watchPlugins: Array<string | [string, Record<string, unknown>]>;
+  workerIdleMemoryLimit: number;
 }>;
 
 export type SnapshotUpdateState = 'all' | 'new' | 'none';
@@ -420,6 +421,7 @@ export type GlobalConfig = {
     path: string;
     config: Record<string, unknown>;
   }> | null;
+  workerIdleMemoryLimit?: number;
 };
 
 export type ProjectConfig = {
@@ -478,6 +480,7 @@ export type ProjectConfig = {
   transformIgnorePatterns: Array<string>;
   watchPathIgnorePatterns: Array<string>;
   unmockedModulePathPatterns?: Array<string>;
+  workerIdleMemoryLimit?: number;
 };
 
 export type Argv = Arguments<
@@ -571,5 +574,6 @@ export type Argv = Arguments<
     watchAll: boolean;
     watchman: boolean;
     watchPathIgnorePatterns: Array<string>;
+    workerIdleMemoryLimit: number;
   }>
 >;

packages/jest-worker/src/__tests__/process-integration.test.js

@@ -6,7 +6,12 @@
  */
 
 import EventEmitter from 'events';
-import {CHILD_MESSAGE_CALL, PARENT_MESSAGE_OK} from '../types';
+import {
+  CHILD_MESSAGE_CALL,
+  CHILD_MESSAGE_MEM_USAGE,
+  PARENT_MESSAGE_OK,
+  WorkerFarmOptions,
+} from '../types';
 
 let Farm;
 let mockForkedProcesses;
@@ -175,4 +180,36 @@ describe('Jest Worker Integration', () => {
     expect(await promise1).toBe('worker-1');
     expect(await promise2).toBe('worker-2');
   });
+
+  it('should check for memory limits', async () => {
+    /** @type WorkerFarmOptions */
+    const options = {
+      computeWorkerKey: () => '1234567890abcdef',
+      exposedMethods: ['foo', 'bar'],
+      idleMemoryLimit: 0.4,
+      numWorkers: 2,
+    };
+
+    const farm = new Farm('/tmp/baz.js', options);
+
+    // Send a call to the farm
+    const promise0 = farm.foo('param-0');
+
+    // Send different responses for each call (from the same child).
+    replySuccess(0, 'worker-0');
+
+    // Check that all the calls have been received by the same child).
+    // We're not using the assertCallsToChild helper because we need to check
+    // for other send types.
+    expect(mockForkedProcesses[0].send).toHaveBeenCalledTimes(3);
+    expect(mockForkedProcesses[0].send.mock.calls[1][0]).toEqual([
+      CHILD_MESSAGE_CALL,
+      true,
+      'foo',
+      ['param-0'],
+    ]);
+    expect(mockForkedProcesses[0].send.mock.calls[2][0]).toEqual([
+      CHILD_MESSAGE_MEM_USAGE,
+    ]);
+  });
 });

packages/jest-worker/src/base/BaseWorkerPool.ts

@@ -40,6 +40,7 @@ export default class BaseWorkerPool {
     for (let i = 0; i < options.numWorkers; i++) {
       const workerOptions: WorkerOptions = {
         forkOptions,
+        idleMemoryLimit: this._options.idleMemoryLimit,
         maxRetries,
         resourceLimits,
         setupArgs,

packages/jest-worker/src/index.ts

@@ -96,6 +96,7 @@ export class Worker {
     const workerPoolOptions: WorkerPoolOptions = {
       enableWorkerThreads: this._options.enableWorkerThreads ?? false,
       forkOptions: this._options.forkOptions ?? {},
+      idleMemoryLimit: this._options.idleMemoryLimit,
       maxRetries: this._options.maxRetries ?? 3,
       numWorkers: this._options.numWorkers ?? Math.max(cpus().length - 1, 1),
       resourceLimits: this._options.resourceLimits ?? {},

packages/jest-worker/src/types.ts

@@ -36,11 +36,13 @@ export type WorkerModule<T> = {
 export const CHILD_MESSAGE_INITIALIZE = 0;
 export const CHILD_MESSAGE_CALL = 1;
 export const CHILD_MESSAGE_END = 2;
+export const CHILD_MESSAGE_MEM_USAGE = 3;
 
 export const PARENT_MESSAGE_OK = 0;
 export const PARENT_MESSAGE_CLIENT_ERROR = 1;
 export const PARENT_MESSAGE_SETUP_ERROR = 2;
 export const PARENT_MESSAGE_CUSTOM = 3;
+export const PARENT_MESSAGE_MEM_USAGE = 4;
 
 export type PARENT_MESSAGE_ERROR =
   | typeof PARENT_MESSAGE_CLIENT_ERROR
@@ -122,6 +124,7 @@ export type WorkerFarmOptions = {
     options?: WorkerPoolOptions,
   ) => WorkerPoolInterface;
   workerSchedulingPolicy?: WorkerSchedulingPolicy;
+  idleMemoryLimit?: number;
 };
 
 export type WorkerPoolOptions = {
@@ -131,6 +134,7 @@ export type WorkerPoolOptions = {
   maxRetries: number;
   numWorkers: number;
   enableWorkerThreads: boolean;
+  idleMemoryLimit?: number;
 };
 
 export type WorkerOptions = {
@@ -141,6 +145,7 @@ export type WorkerOptions = {
   workerId: number;
   workerData?: unknown;
   workerPath: string;
+  idleMemoryLimit?: number;
 };
 
 // Messages passed from the parent to the children.
@@ -174,10 +179,15 @@ export type ChildMessageEnd = [
   boolean, // processed
 ];
 
+export type ChildMessageMemUsage = [
+  typeof CHILD_MESSAGE_MEM_USAGE, // type
+];
+
 export type ChildMessage =
   | ChildMessageInitialize
   | ChildMessageCall
-  | ChildMessageEnd;
+  | ChildMessageEnd
+  | ChildMessageMemUsage;
 
 // Messages passed from the children to the parent.
 
@@ -191,6 +201,11 @@ export type ParentMessageOk = [
   unknown, // result
 ];
 
+export type ParentMessageMemUsage = [
+  typeof PARENT_MESSAGE_MEM_USAGE, // type
+  number, // used memory in bytes
+];
+
 export type ParentMessageError = [
   PARENT_MESSAGE_ERROR, // type
   string, // constructor
@@ -202,7 +217,8 @@ export type ParentMessageError = [
 export type ParentMessage =
   | ParentMessageOk
   | ParentMessageError
-  | ParentMessageCustom;
+  | ParentMessageCustom
+  | ParentMessageMemUsage;
 
 // Queue types.
 

packages/jest-worker/src/workers/ChildProcessWorker.ts

@@ -6,17 +6,20 @@
  */
 
 import {ChildProcess, fork} from 'child_process';
+import {totalmem} from 'os';
 import {PassThrough} from 'stream';
 import mergeStream = require('merge-stream');
 import {stdout as stdoutSupportsColor} from 'supports-color';
 import {
   CHILD_MESSAGE_INITIALIZE,
+  CHILD_MESSAGE_MEM_USAGE,
   ChildMessage,
   OnCustomMessage,
   OnEnd,
   OnStart,
   PARENT_MESSAGE_CLIENT_ERROR,
   PARENT_MESSAGE_CUSTOM,
+  PARENT_MESSAGE_MEM_USAGE,
   PARENT_MESSAGE_OK,
   PARENT_MESSAGE_SETUP_ERROR,
   ParentMessage,
@@ -64,6 +67,9 @@ export default class ChildProcessWorker implements WorkerInterface {
 
   private _exitPromise: Promise<void>;
   private _resolveExitPromise!: () => void;
+  private _childIdleMemoryUsage: number | null;
+  private _childIdleMemoryUsageLimit: number | null;
+  private _restarting = false;
 
   constructor(options: WorkerOptions) {
     this._options = options;
@@ -73,6 +79,8 @@ export default class ChildProcessWorker implements WorkerInterface {
     this._fakeStream = null;
     this._stdout = null;
     this._stderr = null;
+    this._childIdleMemoryUsage = null;
+    this._childIdleMemoryUsageLimit = options.idleMemoryLimit || null;
 
     this._exitPromise = new Promise(resolve => {
       this._resolveExitPromise = resolve;
@@ -82,6 +90,8 @@ export default class ChildProcessWorker implements WorkerInterface {
   }
 
   initialize(): void {
+    this._restarting = false;
+
     const forceColor = stdoutSupportsColor ? {FORCE_COLOR: '1'} : {};
     const child = fork(require.resolve('./processChild'), [], {
       cwd: process.cwd(),
@@ -201,17 +211,44 @@ export default class ChildProcessWorker implements WorkerInterface {
       case PARENT_MESSAGE_CUSTOM:
         this._onCustomMessage(response[1]);
         break;
+
+      case PARENT_MESSAGE_MEM_USAGE:
+        this._childIdleMemoryUsage = response[1];
+
+        this._performRestartIfRequired();
+
+        break;
+
       default:
         throw new TypeError(`Unexpected response from worker: ${response[0]}`);
     }
   }
 
+  private _performRestartIfRequired(): void {
+    let limit = this._childIdleMemoryUsageLimit;
+
+    if (limit && limit > 0 && limit <= 1) {
+      limit = Math.floor(totalmem() * limit);
+    }
+
+    if (
+      limit &&
+      this._childIdleMemoryUsage &&
+      this._childIdleMemoryUsage > limit
+    ) {
+      this._restarting = true;
+
+      this.killChild();
+    }
+  }
+
   private _onExit(exitCode: number | null, signal: NodeJS.Signals | null) {
     if (
-      exitCode !== 0 &&
-      exitCode !== null &&
-      exitCode !== SIGTERM_EXIT_CODE &&
-      exitCode !== SIGKILL_EXIT_CODE
+      (exitCode !== 0 &&
+        exitCode !== null &&
+        exitCode !== SIGTERM_EXIT_CODE &&
+        exitCode !== SIGKILL_EXIT_CODE) ||
+      this._restarting
     ) {
       this.initialize();
 
@@ -241,7 +278,12 @@ export default class ChildProcessWorker implements WorkerInterface {
     onCustomMessage: OnCustomMessage,
   ): void {
     onProcessStart(this);
+
     this._onProcessEnd = (...args) => {
+      if (this._childIdleMemoryUsageLimit) {
+        this._child.send([CHILD_MESSAGE_MEM_USAGE]);
+      }
+
       // Clean the request to avoid sending past requests to workers that fail
       // while waiting for a new request (timers, unhandled rejections...)
       this._request = null;
@@ -260,12 +302,13 @@ export default class ChildProcessWorker implements WorkerInterface {
     return this._exitPromise;
   }
 
-  forceExit(): void {
+  killChild(): NodeJS.Timeout {
     this._child.kill('SIGTERM');
-    const sigkillTimeout = setTimeout(
-      () => this._child.kill('SIGKILL'),
-      SIGKILL_DELAY,
-    );
+    return setTimeout(() => this._child.kill('SIGKILL'), SIGKILL_DELAY);
+  }
+
+  forceExit(): void {
+    const sigkillTimeout = this.killChild();
     this._exitPromise.then(() => clearTimeout(sigkillTimeout));
   }