From a2326a7c84633c57e5b52f67398317787b555acc Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Tue, 1 Mar 2022 16:02:00 +0530 Subject: [PATCH 01/39] fix(websocketshard): deal with zombie connection caused by 4009 --- src/client/websocket/WebSocketShard.js | 198 +++++++++++++++++++++++-- src/util/Constants.js | 1 + typings/index.d.ts | 5 + 3 files changed, 188 insertions(+), 16 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 35174f6d88ff..3cd85f92649d 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -1,7 +1,9 @@ +/* eslint-disable no-console */ 'use strict'; const EventEmitter = require('node:events'); -const { setTimeout, setInterval } = require('node:timers'); +const { setTimeout, setInterval, clearTimeout } = require('node:timers'); +const { setTimeout: sleep } = require('node:timers/promises'); const WebSocket = require('../../WebSocket'); const { Status, Events, ShardEvents, Opcodes, WSEvents } = require('../../util/Constants'); const Intents = require('../../util/Intents'); @@ -81,6 +83,13 @@ class WebSocketShard extends EventEmitter { */ this.lastHeartbeatAcked = true; + /** + * Used to prevent from calling the onClose event twice while closing or terminating the WebSocket. + * @type {boolean} + * @private + */ + this.closeEmitted = false; + /** * Contains the rate limit queue and metadata * @name WebSocketShard#ratelimit @@ -126,6 +135,14 @@ class WebSocketShard extends EventEmitter { */ Object.defineProperty(this, 'helloTimeout', { value: null, writable: true }); + /** + * The WebSocket timeout. + * @name WebSocketShard#WsCloseTimeout + * @type {?NodeJS.Timeout} + * @private + */ + Object.defineProperty(this, 'WsCloseTimeout', { value: null, writable: true }); + /** * If the manager attached its event handlers on the shard * @name WebSocketShard#eventsAttached @@ -188,6 +205,7 @@ class WebSocketShard extends EventEmitter { this.removeListener(ShardEvents.RESUMED, onResumed); this.removeListener(ShardEvents.INVALID_SESSION, onInvalidOrDestroyed); this.removeListener(ShardEvents.DESTROYED, onInvalidOrDestroyed); + this.removeListener(ShardEvents.ZOMBIE_CONNECTION, this.handleZombieConnection); }; const onReady = () => { @@ -253,7 +271,8 @@ class WebSocketShard extends EventEmitter { this.connectedAt = Date.now(); - const ws = (this.connection = WebSocket.create(gateway, wsQuery)); + // Adding a handshake timeout to just make sure no zombie connection appears. + const ws = (this.connection = WebSocket.create(gateway, wsQuery, { handshakeTimeout: 30000 })); ws.onopen = this.onOpen.bind(this); ws.onmessage = this.onMessage.bind(this); ws.onerror = this.onError.bind(this); @@ -351,10 +370,25 @@ class WebSocketShard extends EventEmitter { this.setHeartbeatTimer(-1); this.setHelloTimeout(-1); // If we still have a connection object, clean up its listeners - if (this.connection) this._cleanupConnection(); + if (this.connection) { + this.debug('[WebSocket] onClose: Cleaning up the Connection.'); + this._cleanupConnection(); + } + this.closeEmitted = true; + this.status = Status.DISCONNECTED; + // Step 1: Null the connection object + this.debug('Step 1: Null the connection object.'); + this.connection = null; + + this.debug('Step 2: Set the shard status to DISCONNECTED'); + // Step 2: Set the shard status to DISCONNECTED this.status = Status.DISCONNECTED; + this.debug('Step 3: Cache the old sequence (use to attempt a resume)'); + // Step 3: Cache the old sequence (use to attempt a resume) + if (this.sequence !== -1) this.closeSequence = this.sequence; + /** * Emitted when a shard's WebSocket closes. * @private @@ -519,10 +553,51 @@ class WebSocketShard extends EventEmitter { this.debug('Setting a HELLO timeout for 20s.'); this.helloTimeout = setTimeout(() => { this.debug('Did not receive HELLO in time. Destroying and connecting again.'); - this.destroy({ reset: true, closeCode: 4009 }); + this.destroy({ closeCode: 4009 }); }, 20_000).unref(); } + /** + * Sets the WebSocket Close timeout. + * This method is responsilble to detect any zombie connections if the ws fails to close properly, + * Wait for 6s for the ws#close event after ws.close() or ws.terminate() is called. + * @param {number} [time] If set to -1, it will clear the timeout + * @private + */ + setWsCloseTimeout(time) { + if (time === -1) { + if (this.WsCloseTimeout) { + this.debug('Clearing the WebSocket Close timeout.'); + clearTimeout(this.WsCloseTimeout); + this.WsCloseTimeout = null; + } + return; + } + this.WsCloseTimeout = setTimeout(() => { + this.debug(`[WebSocket] Close Emitted: ${this.closeEmitted}`); + // Check connection is null or if close event was emitted. + if (!this.connection || this.closeEmitted) { + this.debug( + `[WebSocket] WebSocket close not detected. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? 3] + } | Close Emitted: ${this.closeEmitted}`, + ); + this.closeEmitted = false; + this.setWsCloseTimeout(-1); + return; + } + console.log('Should emit zombieConnection'); + // Waiting for approx 5s. + + /** + * Emitted when a shard's WebSocket is not closed in time. + * @private + * @event WebSocketShard#zombieConnection + */ + this.emit(ShardEvents.ZOMBIE_CONNECTION); + }, 6000).unref(); + } + /** * Sets the heartbeat timer for this shard. * @param {number} time If -1, clears the interval, any other number sets an interval @@ -563,8 +638,7 @@ class WebSocketShard extends EventEmitter { Sequence : ${this.sequence} Connection State: ${this.connection ? CONNECTION_STATE[this.connection.readyState] : 'No Connection??'}`, ); - - this.destroy({ closeCode: 4009, reset: true }); + this.destroy({ closeCode: 4009 }); return; } @@ -702,6 +776,9 @@ class WebSocketShard extends EventEmitter { * @private */ destroy({ closeCode = 1_000, reset = false, emit = true, log = true } = {}) { + // Making the variable false to check for zombie connections. + this.closeEmitted = false; + if (log) { this.debug(`[DESTROY] Close Code : ${closeCode} @@ -709,24 +786,39 @@ class WebSocketShard extends EventEmitter { Emit DESTROYED: ${emit}`); } + this.debug(`[WS Destroy] Step 0: Remove all timers.`); // Step 0: Remove all timers this.setHeartbeatTimer(-1); this.setHelloTimeout(-1); + this.debug( + `[WS Destroy] Step 1: Attempting to close the WebSocket. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? 3] + }`, + ); // Step 1: Close the WebSocket connection, if any, otherwise, emit DESTROYED if (this.connection) { // If the connection is currently opened, we will (hopefully) receive close if (this.connection.readyState === WebSocket.OPEN) { this.connection.close(closeCode); + this.debug( + `[WebSocket] Close: Tried closing. | WS State: ${CONNECTION_STATE[this.connection?.readyState ?? 3]}`, + ); } else { // Connection is not OPEN - this.debug(`WS State: ${CONNECTION_STATE[this.connection.readyState]}`); + this.debug(`WS State: ${CONNECTION_STATE[this.connection?.readyState ?? 3]}`); // Remove listeners from the connection this._cleanupConnection(); // Attempt to close the connection just in case try { this.connection.close(closeCode); } catch { + this.debug( + `[WebSocket] Close: Something went wrong while closing the WebSocket, Now calling terminate. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? 3] + }`, + ); + this.connection.terminate(); // No-op } // Emit the destroyed event if needed @@ -737,30 +829,104 @@ class WebSocketShard extends EventEmitter { this._emitDestroyed(); } - // Step 2: Null the connection object - this.connection = null; - - // Step 3: Set the shard status to DISCONNECTED - this.status = Status.DISCONNECTED; + /** + * Just to make sure the readyState is not stuck at CLOSING incase of a closeCode 4009. + * we can use this for other closeCodes aswell but rightnow so far only 4009 is failing to close. + **/ + if (closeCode === 4009) { + this.debug( + `[WebSocket] closeCode[4009]: Adding a timeout to check the connection state. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? 3] + }`, + ); + this.setWsCloseTimeout(); + } - // Step 4: Cache the old sequence (use to attempt a resume) - if (this.sequence !== -1) this.closeSequence = this.sequence; + this.debug( + `[WS Destroy] Step 2: Resetting the sequence and session id if requested. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? 3] + }`, + ); - // Step 5: Reset the sequence and session id if requested + // Step 2: Reset the sequence and session id if requested if (reset) { this.sequence = -1; this.sessionId = null; } - // Step 6: reset the rate limit data + // Step 3: Watch for zombie connection event which could be emitted when step 1 fails to close the WebSocket + this.on(ShardEvents.ZOMBIE_CONNECTION, this.handleZombieConnection); + + this.debug('Step 4: reset the rate limit data'); + // Step 4: reset the rate limit data this.ratelimit.remaining = this.ratelimit.total; this.ratelimit.queue.length = 0; if (this.ratelimit.timer) { + this.debug(`[WS Destroy] Step 4: Clearing ratelimit timer`); clearTimeout(this.ratelimit.timer); this.ratelimit.timer = null; } } + /** + * Forcefully closes the WebSocket when the destory() method could not close it normally + * using the ws.close(); and ws.terminate(); + * @private + * @returns {Promise} + */ + async handleZombieConnection() { + // Continuing from step 2 from the destroy method. + + // Step 3 Check if the connection was closed and readyState is at CLOSED. If not close it. + if (this.connection?.readyState === WebSocket.CLOSING || this.connection?.readyState === WebSocket.OPEN) { + this.debug( + `[WS Destroy] Step 3: WebSocket Still Connected: trying to terminate. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? 3] + }`, + ); + + // Trying to close WebSocket with terminate.. + try { + this.connection.terminate(); + } catch { + // No op. + } + + // Wait for 400ms for ws to close. + await sleep(400); + this.debug(`[WebSocket] Final Close Check. | WS State: ${CONNECTION_STATE[this.connection?.readyState]}`); + if (this.connection?.readyState === WebSocket.CLOSING || this.connection?.readyState === WebSocket.OPEN) { + this.debug( + // eslint-disable-next-line max-len + `[WebSocket] Connection still stuck at Closing: Calling a Manual Destroy of the socket to Close and reconnect. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? 3] + }`, + ); + + /** + * This is an important step to deal with zombie connections where in shard never reconnects + * after a 4009 closeCode due to WebSocket being stuck at CLOSING ready state. + * Check the issue https://github.com/discordjs/discord.js/issues/7450 + * + * The _socket.destroy() method Ensures that no more I/O activity happens on this socket. + * Destroys the stream and closes the connection. Refer: https://nodejs.org/api/net.html#socketdestroy + * _socket.destroy() is also being invoked in ws.terminate() method. + * ._socket.destroy emits the close event and makes the readyState to CLOSED. + */ + + // manual destory + this.connection._socket.destroy(); + this.connection.emitClose(); + + this.debug( + `[WebSocket] Applied a Manual Destroy, Clearing the WS State Logger in 5s. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? 3] + }`, + ); + } + } + } + /** * Cleans up the WebSocket connection listeners. * @private diff --git a/src/util/Constants.js b/src/util/Constants.js index 14b0e526c906..accfe2fd8b8a 100644 --- a/src/util/Constants.js +++ b/src/util/Constants.js @@ -218,6 +218,7 @@ exports.ShardEvents = { READY: 'ready', RESUMED: 'resumed', ALL_READY: 'allReady', + ZOMBIE_CONNECTION: 'zombieConnection', }; /** diff --git a/typings/index.d.ts b/typings/index.d.ts index e628956c51d8..51106083f78f 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -2608,6 +2608,7 @@ export interface WebSocketShardEvents { invalidSession: []; close: [event: CloseEvent]; allReady: [unavailableGuilds?: Set]; + zombieConnection: []; } export class WebSocketShard extends EventEmitter { @@ -2623,6 +2624,8 @@ export class WebSocketShard extends EventEmitter { private eventsAttached: boolean; private expectedGuilds: Set | null; private readyTimeout: NodeJS.Timeout | null; + private closeEmitted: boolean; + private WsCloseTimeout: NodeJS.Timeout | null; public manager: WebSocketManager; public id: number; @@ -2638,6 +2641,7 @@ export class WebSocketShard extends EventEmitter { private onPacket(packet: unknown): void; private checkReady(): void; private setHelloTimeout(time?: number): void; + private setWsCloseTimeout(time?: number): void; private setHeartbeatTimer(time: number): void; private sendHeartbeat(): void; private ackHeartbeat(): void; @@ -2647,6 +2651,7 @@ export class WebSocketShard extends EventEmitter { private _send(data: unknown): void; private processQueue(): void; private destroy(destroyOptions?: { closeCode?: number; reset?: boolean; emit?: boolean; log?: boolean }): void; + private handleZombieConnection(): Promise; private _cleanupConnection(): void; private _emitDestroyed(): void; From aab64b8d258508f6d23ee80fc3fffea87a0fe3c3 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Tue, 1 Mar 2022 16:11:41 +0530 Subject: [PATCH 02/39] chore: remove eslint-disable for console --- src/client/websocket/WebSocketShard.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 3cd85f92649d..88dd13bae8b7 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -1,4 +1,3 @@ -/* eslint-disable no-console */ 'use strict'; const EventEmitter = require('node:events'); @@ -586,7 +585,6 @@ class WebSocketShard extends EventEmitter { this.setWsCloseTimeout(-1); return; } - console.log('Should emit zombieConnection'); // Waiting for approx 5s. /** From 25ac5ea84a660a9279e97de33cc445deaf5ace4b Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Tue, 1 Mar 2022 16:55:12 +0530 Subject: [PATCH 03/39] fix: use enum instead of using numbers --- src/client/websocket/WebSocketShard.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 88dd13bae8b7..b716c444d65d 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -578,7 +578,7 @@ class WebSocketShard extends EventEmitter { if (!this.connection || this.closeEmitted) { this.debug( `[WebSocket] WebSocket close not detected. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? 3] + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] } | Close Emitted: ${this.closeEmitted}`, ); this.closeEmitted = false; @@ -791,7 +791,7 @@ class WebSocketShard extends EventEmitter { this.debug( `[WS Destroy] Step 1: Attempting to close the WebSocket. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? 3] + CONNECTION_STATE[this.connection?.readyState ?? CONNECTION_STATE] }`, ); // Step 1: Close the WebSocket connection, if any, otherwise, emit DESTROYED @@ -800,11 +800,13 @@ class WebSocketShard extends EventEmitter { if (this.connection.readyState === WebSocket.OPEN) { this.connection.close(closeCode); this.debug( - `[WebSocket] Close: Tried closing. | WS State: ${CONNECTION_STATE[this.connection?.readyState ?? 3]}`, + `[WebSocket] Close: Tried closing. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] + }`, ); } else { // Connection is not OPEN - this.debug(`WS State: ${CONNECTION_STATE[this.connection?.readyState ?? 3]}`); + this.debug(`WS State: ${CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED]}`); // Remove listeners from the connection this._cleanupConnection(); // Attempt to close the connection just in case @@ -813,7 +815,7 @@ class WebSocketShard extends EventEmitter { } catch { this.debug( `[WebSocket] Close: Something went wrong while closing the WebSocket, Now calling terminate. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? 3] + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); this.connection.terminate(); @@ -834,7 +836,7 @@ class WebSocketShard extends EventEmitter { if (closeCode === 4009) { this.debug( `[WebSocket] closeCode[4009]: Adding a timeout to check the connection state. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? 3] + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); this.setWsCloseTimeout(); @@ -842,7 +844,7 @@ class WebSocketShard extends EventEmitter { this.debug( `[WS Destroy] Step 2: Resetting the sequence and session id if requested. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? 3] + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); @@ -879,7 +881,7 @@ class WebSocketShard extends EventEmitter { if (this.connection?.readyState === WebSocket.CLOSING || this.connection?.readyState === WebSocket.OPEN) { this.debug( `[WS Destroy] Step 3: WebSocket Still Connected: trying to terminate. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? 3] + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); @@ -897,7 +899,7 @@ class WebSocketShard extends EventEmitter { this.debug( // eslint-disable-next-line max-len `[WebSocket] Connection still stuck at Closing: Calling a Manual Destroy of the socket to Close and reconnect. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? 3] + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); @@ -918,7 +920,7 @@ class WebSocketShard extends EventEmitter { this.debug( `[WebSocket] Applied a Manual Destroy, Clearing the WS State Logger in 5s. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? 3] + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); } From 48478980c6a7190b5bf8ffce726fc53bcc294c34 Mon Sep 17 00:00:00 2001 From: Voxelli <69213593+legendhimslef@users.noreply.github.com> Date: Tue, 1 Mar 2022 17:13:26 +0530 Subject: [PATCH 04/39] Apply suggestions from code review Co-authored-by: Almeida --- src/client/websocket/WebSocketShard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index b716c444d65d..1b5fa88cbe43 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -791,7 +791,7 @@ class WebSocketShard extends EventEmitter { this.debug( `[WS Destroy] Step 1: Attempting to close the WebSocket. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? CONNECTION_STATE] + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); // Step 1: Close the WebSocket connection, if any, otherwise, emit DESTROYED From 52b0abd07e59bc83e375756b31c52efe85c88391 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Tue, 1 Mar 2022 17:21:19 +0530 Subject: [PATCH 05/39] fix: apply suggestions --- src/client/websocket/WebSocketShard.js | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 1b5fa88cbe43..4bdf7b0e969e 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -799,14 +799,10 @@ class WebSocketShard extends EventEmitter { // If the connection is currently opened, we will (hopefully) receive close if (this.connection.readyState === WebSocket.OPEN) { this.connection.close(closeCode); - this.debug( - `[WebSocket] Close: Tried closing. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] - }`, - ); + this.debug(`[WebSocket] Close: Tried closing. | WS State: ${CONNECTION_STATE[this.connection.readyState]}`); } else { // Connection is not OPEN - this.debug(`WS State: ${CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED]}`); + this.debug(`WS State: ${CONNECTION_STATE[this.connection.readyState]}`); // Remove listeners from the connection this._cleanupConnection(); // Attempt to close the connection just in case @@ -815,7 +811,7 @@ class WebSocketShard extends EventEmitter { } catch { this.debug( `[WebSocket] Close: Something went wrong while closing the WebSocket, Now calling terminate. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] + CONNECTION_STATE[this.connection.readyState] }`, ); this.connection.terminate(); @@ -881,7 +877,7 @@ class WebSocketShard extends EventEmitter { if (this.connection?.readyState === WebSocket.CLOSING || this.connection?.readyState === WebSocket.OPEN) { this.debug( `[WS Destroy] Step 3: WebSocket Still Connected: trying to terminate. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] + CONNECTION_STATE[this.connection.readyState] }`, ); @@ -894,12 +890,12 @@ class WebSocketShard extends EventEmitter { // Wait for 400ms for ws to close. await sleep(400); - this.debug(`[WebSocket] Final Close Check. | WS State: ${CONNECTION_STATE[this.connection?.readyState]}`); - if (this.connection?.readyState === WebSocket.CLOSING || this.connection?.readyState === WebSocket.OPEN) { + this.debug(`[WebSocket] Final Close Check. | WS State: ${CONNECTION_STATE[this.connection.readyState]}`); + if (this.connection.readyState === WebSocket.CLOSING || this.connection.readyState === WebSocket.OPEN) { this.debug( // eslint-disable-next-line max-len `[WebSocket] Connection still stuck at Closing: Calling a Manual Destroy of the socket to Close and reconnect. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] + CONNECTION_STATE[this.connection.readyState] }`, ); @@ -920,7 +916,7 @@ class WebSocketShard extends EventEmitter { this.debug( `[WebSocket] Applied a Manual Destroy, Clearing the WS State Logger in 5s. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] + CONNECTION_STATE[this.connection.readyState] }`, ); } From ac2beb3335c6b262b223b1c077e4250cc546018f Mon Sep 17 00:00:00 2001 From: Voxelli <69213593+legendhimslef@users.noreply.github.com> Date: Tue, 1 Mar 2022 17:28:40 +0530 Subject: [PATCH 06/39] Apply suggestions from code review Co-authored-by: Vitor --- src/client/websocket/WebSocketShard.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 4bdf7b0e969e..16da06a39bda 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -865,7 +865,7 @@ class WebSocketShard extends EventEmitter { } /** - * Forcefully closes the WebSocket when the destory() method could not close it normally + * Forcefully closes the WebSocket when the destroy() method could not close it normally * using the ws.close(); and ws.terminate(); * @private * @returns {Promise} @@ -904,10 +904,10 @@ class WebSocketShard extends EventEmitter { * after a 4009 closeCode due to WebSocket being stuck at CLOSING ready state. * Check the issue https://github.com/discordjs/discord.js/issues/7450 * - * The _socket.destroy() method Ensures that no more I/O activity happens on this socket. + * The _socket.destroy() method ensures that no more I/O activity happens on this socket. * Destroys the stream and closes the connection. Refer: https://nodejs.org/api/net.html#socketdestroy * _socket.destroy() is also being invoked in ws.terminate() method. - * ._socket.destroy emits the close event and makes the readyState to CLOSED. + * _socket.destroy() emits the close event and makes the readyState to CLOSED. */ // manual destory From 2cb93a6baca83b2efcbdc6db66a0e01169aa2c0c Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Tue, 1 Mar 2022 17:49:50 +0530 Subject: [PATCH 07/39] fix: remove extra reassignment of status --- src/client/websocket/WebSocketShard.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 4bdf7b0e969e..dcf1639a59bd 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -374,7 +374,6 @@ class WebSocketShard extends EventEmitter { this._cleanupConnection(); } this.closeEmitted = true; - this.status = Status.DISCONNECTED; // Step 1: Null the connection object this.debug('Step 1: Null the connection object.'); From 78f84f93d002e23b4709903ffa1d19d308198ecd Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Tue, 1 Mar 2022 21:39:43 +0530 Subject: [PATCH 08/39] fix: clear timeout on ws close --- src/client/websocket/WebSocketShard.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 89f7b1f62fa8..5d15e7a3bbc5 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -374,6 +374,8 @@ class WebSocketShard extends EventEmitter { this._cleanupConnection(); } this.closeEmitted = true; + // Clearing the websocket close timeout as close was emitted. + this.setWsCloseTimeout(-1); // Step 1: Null the connection object this.debug('Step 1: Null the connection object.'); @@ -565,7 +567,7 @@ class WebSocketShard extends EventEmitter { setWsCloseTimeout(time) { if (time === -1) { if (this.WsCloseTimeout) { - this.debug('Clearing the WebSocket Close timeout.'); + this.debug('Clearing the WebSocket close timeout.'); clearTimeout(this.WsCloseTimeout); this.WsCloseTimeout = null; } @@ -576,7 +578,7 @@ class WebSocketShard extends EventEmitter { // Check connection is null or if close event was emitted. if (!this.connection || this.closeEmitted) { this.debug( - `[WebSocket] WebSocket close not detected. | WS State: ${ + `[WebSocket] was closed. | WS State: ${ CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] } | Close Emitted: ${this.closeEmitted}`, ); @@ -584,8 +586,11 @@ class WebSocketShard extends EventEmitter { this.setWsCloseTimeout(-1); return; } - // Waiting for approx 5s. + this.debug( + `[WebSocket] did not close properly, Assuming a zombie connection. Destroying and reconnecting again. + WS State: ${CONNECTION_STATE[this.connection.readyState]} | Close Emitted: ${this.closeEmitted}`, + ); /** * Emitted when a shard's WebSocket is not closed in time. * @private From e80a8676bbe65f4f6c664edd61a610d9dc0b19b1 Mon Sep 17 00:00:00 2001 From: Voxelli <69213593+legendhimslef@users.noreply.github.com> Date: Sun, 6 Mar 2022 18:32:00 +0530 Subject: [PATCH 09/39] Apply suggestions from code review Co-authored-by: SpaceEEC --- src/client/websocket/WebSocketShard.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 5d15e7a3bbc5..bf8441dd4b10 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -271,7 +271,7 @@ class WebSocketShard extends EventEmitter { this.connectedAt = Date.now(); // Adding a handshake timeout to just make sure no zombie connection appears. - const ws = (this.connection = WebSocket.create(gateway, wsQuery, { handshakeTimeout: 30000 })); + const ws = (this.connection = WebSocket.create(gateway, wsQuery, { handshakeTimeout: 30_000 })); ws.onopen = this.onOpen.bind(this); ws.onmessage = this.onMessage.bind(this); ws.onerror = this.onError.bind(this); @@ -588,7 +588,7 @@ class WebSocketShard extends EventEmitter { } this.debug( - `[WebSocket] did not close properly, Assuming a zombie connection. Destroying and reconnecting again. + `[WebSocket] did not close properly, assuming a zombie connection. Destroying and reconnecting again. WS State: ${CONNECTION_STATE[this.connection.readyState]} | Close Emitted: ${this.closeEmitted}`, ); /** @@ -597,7 +597,7 @@ class WebSocketShard extends EventEmitter { * @event WebSocketShard#zombieConnection */ this.emit(ShardEvents.ZOMBIE_CONNECTION); - }, 6000).unref(); + }, 6_000).unref(); } /** From 78b15bc641eb60beea52bf055f12ead3b4f03d33 Mon Sep 17 00:00:00 2001 From: Voxelli <69213593+legendhimslef@users.noreply.github.com> Date: Sun, 6 Mar 2022 19:08:23 +0530 Subject: [PATCH 10/39] Apply suggestions from code review Co-authored-by: Vlad Frangu --- src/client/websocket/WebSocketShard.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index bf8441dd4b10..cf624ea2b8d2 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -794,7 +794,7 @@ class WebSocketShard extends EventEmitter { this.setHelloTimeout(-1); this.debug( - `[WS Destroy] Step 1: Attempting to close the WebSocket. | WS State: ${ + `[WS Destroy] Attempting to close the WebSocket. | WS State: ${ CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); @@ -812,9 +812,9 @@ class WebSocketShard extends EventEmitter { // Attempt to close the connection just in case try { this.connection.close(closeCode); - } catch { + } catch (err) { this.debug( - `[WebSocket] Close: Something went wrong while closing the WebSocket, Now calling terminate. | WS State: ${ + `[WebSocket] Close: Something went wrong while closing the WebSocket: ${err.message || err}. Forcefully terminating the connection | WS State: ${ CONNECTION_STATE[this.connection.readyState] }`, ); @@ -831,11 +831,11 @@ class WebSocketShard extends EventEmitter { /** * Just to make sure the readyState is not stuck at CLOSING incase of a closeCode 4009. - * we can use this for other closeCodes aswell but rightnow so far only 4009 is failing to close. + * we can use this for other close codes as well but right now so far only 4009 is failing to close. **/ if (closeCode === 4009) { this.debug( - `[WebSocket] closeCode[4009]: Adding a timeout to check the connection state. | WS State: ${ + `[WebSocket] Encountered close code 4009. Adding a timeout to check the connection actually closed. | WS State: ${ CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); @@ -898,7 +898,7 @@ class WebSocketShard extends EventEmitter { if (this.connection.readyState === WebSocket.CLOSING || this.connection.readyState === WebSocket.OPEN) { this.debug( // eslint-disable-next-line max-len - `[WebSocket] Connection still stuck at Closing: Calling a Manual Destroy of the socket to Close and reconnect. | WS State: ${ + `[WebSocket] Connection still stuck at Closing: calling a manual destroy of the socket to close and reconnect. | WS State: ${ CONNECTION_STATE[this.connection.readyState] }`, ); @@ -919,7 +919,7 @@ class WebSocketShard extends EventEmitter { this.connection.emitClose(); this.debug( - `[WebSocket] Applied a Manual Destroy, Clearing the WS State Logger in 5s. | WS State: ${ + `[WebSocket] Manually closed the connection. | WS State: ${ CONNECTION_STATE[this.connection.readyState] }`, ); From c154cebff8387308099fcfd8d83c23473ea9aed1 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Sun, 6 Mar 2022 19:24:31 +0530 Subject: [PATCH 11/39] refactor: apply suggestions from code review --- src/client/websocket/WebSocketShard.js | 41 ++++++++------------------ typings/index.d.ts | 2 +- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index cf624ea2b8d2..2bd118ee7a92 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -136,11 +136,11 @@ class WebSocketShard extends EventEmitter { /** * The WebSocket timeout. - * @name WebSocketShard#WsCloseTimeout + * @name WebSocketShard#wsCloseTimeout * @type {?NodeJS.Timeout} * @private */ - Object.defineProperty(this, 'WsCloseTimeout', { value: null, writable: true }); + Object.defineProperty(this, 'wsCloseTimeout', { value: null, writable: true }); /** * If the manager attached its event handlers on the shard @@ -369,23 +369,18 @@ class WebSocketShard extends EventEmitter { this.setHeartbeatTimer(-1); this.setHelloTimeout(-1); // If we still have a connection object, clean up its listeners - if (this.connection) { - this.debug('[WebSocket] onClose: Cleaning up the Connection.'); - this._cleanupConnection(); - } + if (this.connection) this._cleanupConnection(); + this.closeEmitted = true; // Clearing the websocket close timeout as close was emitted. this.setWsCloseTimeout(-1); // Step 1: Null the connection object - this.debug('Step 1: Null the connection object.'); this.connection = null; - this.debug('Step 2: Set the shard status to DISCONNECTED'); // Step 2: Set the shard status to DISCONNECTED this.status = Status.DISCONNECTED; - this.debug('Step 3: Cache the old sequence (use to attempt a resume)'); // Step 3: Cache the old sequence (use to attempt a resume) if (this.sequence !== -1) this.closeSequence = this.sequence; @@ -566,14 +561,14 @@ class WebSocketShard extends EventEmitter { */ setWsCloseTimeout(time) { if (time === -1) { - if (this.WsCloseTimeout) { + if (this.wsCloseTimeout) { this.debug('Clearing the WebSocket close timeout.'); - clearTimeout(this.WsCloseTimeout); - this.WsCloseTimeout = null; + clearTimeout(this.wsCloseTimeout); + this.wsCloseTimeout = null; } return; } - this.WsCloseTimeout = setTimeout(() => { + this.wsCloseTimeout = setTimeout(() => { this.debug(`[WebSocket] Close Emitted: ${this.closeEmitted}`); // Check connection is null or if close event was emitted. if (!this.connection || this.closeEmitted) { @@ -788,7 +783,6 @@ class WebSocketShard extends EventEmitter { Emit DESTROYED: ${emit}`); } - this.debug(`[WS Destroy] Step 0: Remove all timers.`); // Step 0: Remove all timers this.setHeartbeatTimer(-1); this.setHelloTimeout(-1); @@ -814,9 +808,9 @@ class WebSocketShard extends EventEmitter { this.connection.close(closeCode); } catch (err) { this.debug( - `[WebSocket] Close: Something went wrong while closing the WebSocket: ${err.message || err}. Forcefully terminating the connection | WS State: ${ - CONNECTION_STATE[this.connection.readyState] - }`, + `[WebSocket] Close: Something went wrong while closing the WebSocket: ${ + err.message || err + }. Forcefully terminating the connection | WS State: ${CONNECTION_STATE[this.connection.readyState]}`, ); this.connection.terminate(); // No-op @@ -835,6 +829,7 @@ class WebSocketShard extends EventEmitter { **/ if (closeCode === 4009) { this.debug( + // eslint-disable-next-line max-len `[WebSocket] Encountered close code 4009. Adding a timeout to check the connection actually closed. | WS State: ${ CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, @@ -842,12 +837,6 @@ class WebSocketShard extends EventEmitter { this.setWsCloseTimeout(); } - this.debug( - `[WS Destroy] Step 2: Resetting the sequence and session id if requested. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] - }`, - ); - // Step 2: Reset the sequence and session id if requested if (reset) { this.sequence = -1; @@ -857,12 +846,10 @@ class WebSocketShard extends EventEmitter { // Step 3: Watch for zombie connection event which could be emitted when step 1 fails to close the WebSocket this.on(ShardEvents.ZOMBIE_CONNECTION, this.handleZombieConnection); - this.debug('Step 4: reset the rate limit data'); // Step 4: reset the rate limit data this.ratelimit.remaining = this.ratelimit.total; this.ratelimit.queue.length = 0; if (this.ratelimit.timer) { - this.debug(`[WS Destroy] Step 4: Clearing ratelimit timer`); clearTimeout(this.ratelimit.timer); this.ratelimit.timer = null; } @@ -919,9 +906,7 @@ class WebSocketShard extends EventEmitter { this.connection.emitClose(); this.debug( - `[WebSocket] Manually closed the connection. | WS State: ${ - CONNECTION_STATE[this.connection.readyState] - }`, + `[WebSocket] Manually closed the connection. | WS State: ${CONNECTION_STATE[this.connection.readyState]}`, ); } } diff --git a/typings/index.d.ts b/typings/index.d.ts index cc9ddc7162f7..a836c7fe4829 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -2635,7 +2635,7 @@ export class WebSocketShard extends EventEmitter { private expectedGuilds: Set | null; private readyTimeout: NodeJS.Timeout | null; private closeEmitted: boolean; - private WsCloseTimeout: NodeJS.Timeout | null; + private wsCloseTimeout: NodeJS.Timeout | null; public manager: WebSocketManager; public id: number; From d3dfbac891333cdcf21de925bda81e49a41bbfa6 Mon Sep 17 00:00:00 2001 From: Voxelli <69213593+legendhimslef@users.noreply.github.com> Date: Sun, 6 Mar 2022 23:38:04 +0530 Subject: [PATCH 12/39] Apply suggestions from code review Co-authored-by: Jiralite <33201955+Jiralite@users.noreply.github.com> --- src/client/websocket/WebSocketShard.js | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 2bd118ee7a92..eb553574c22a 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -83,7 +83,7 @@ class WebSocketShard extends EventEmitter { this.lastHeartbeatAcked = true; /** - * Used to prevent from calling the onClose event twice while closing or terminating the WebSocket. + * Used to prevent calling {@link WebSocketShard#event:close} twice while closing or terminating the WebSocket. * @type {boolean} * @private */ @@ -372,7 +372,7 @@ class WebSocketShard extends EventEmitter { if (this.connection) this._cleanupConnection(); this.closeEmitted = true; - // Clearing the websocket close timeout as close was emitted. + // Clearing the WebSocket close timeout as close was emitted. this.setWsCloseTimeout(-1); // Step 1: Null the connection object @@ -554,8 +554,7 @@ class WebSocketShard extends EventEmitter { /** * Sets the WebSocket Close timeout. - * This method is responsilble to detect any zombie connections if the ws fails to close properly, - * Wait for 6s for the ws#close event after ws.close() or ws.terminate() is called. + * This method is responsible for detecting any zombie connections if the WebSocket fails to close properly. * @param {number} [time] If set to -1, it will clear the timeout * @private */ @@ -823,9 +822,9 @@ class WebSocketShard extends EventEmitter { this._emitDestroyed(); } - /** - * Just to make sure the readyState is not stuck at CLOSING incase of a closeCode 4009. - * we can use this for other close codes as well but right now so far only 4009 is failing to close. + /* + * Just to make sure the readyState is not stuck at CLOSING in case of a closeCode 4009. + * We can use this for other close codes as well but so far only 4009 is failing to close. **/ if (closeCode === 4009) { this.debug( @@ -846,7 +845,7 @@ class WebSocketShard extends EventEmitter { // Step 3: Watch for zombie connection event which could be emitted when step 1 fails to close the WebSocket this.on(ShardEvents.ZOMBIE_CONNECTION, this.handleZombieConnection); - // Step 4: reset the rate limit data + // Step 4: Reset the rate limit data this.ratelimit.remaining = this.ratelimit.total; this.ratelimit.queue.length = 0; if (this.ratelimit.timer) { @@ -856,8 +855,7 @@ class WebSocketShard extends EventEmitter { } /** - * Forcefully closes the WebSocket when the destroy() method could not close it normally - * using the ws.close(); and ws.terminate(); + * Forcefully closes the WebSocket when {@link WebSocketShard#destroy} could not close it properly. * @private * @returns {Promise} */ @@ -890,9 +888,9 @@ class WebSocketShard extends EventEmitter { }`, ); - /** - * This is an important step to deal with zombie connections where in shard never reconnects - * after a 4009 closeCode due to WebSocket being stuck at CLOSING ready state. + /* + * This is an important step to deal with zombie connections wherein shards never reconnect + * after a 4009 closeCode due to the WebSocket being stuck at the CLOSING ready state. * Check the issue https://github.com/discordjs/discord.js/issues/7450 * * The _socket.destroy() method ensures that no more I/O activity happens on this socket. From fb3ab02b4eeee16403daf3c627a21e91f98d431e Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Mon, 7 Mar 2022 09:44:53 +0530 Subject: [PATCH 13/39] fix: close emitting twice --- src/client/websocket/WebSocketShard.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index eb553574c22a..456f51e0aca0 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -582,8 +582,10 @@ class WebSocketShard extends EventEmitter { } this.debug( - `[WebSocket] did not close properly, assuming a zombie connection. Destroying and reconnecting again. - WS State: ${CONNECTION_STATE[this.connection.readyState]} | Close Emitted: ${this.closeEmitted}`, + // eslint-disable-next-line max-len + `[WebSocket] did not close properly, assuming a zombie connection. Destroying and reconnecting again. WS State: ${ + CONNECTION_STATE[this.connection.readyState] + } | Close Emitted: ${this.closeEmitted}`, ); /** * Emitted when a shard's WebSocket is not closed in time. @@ -901,7 +903,8 @@ class WebSocketShard extends EventEmitter { // manual destory this.connection._socket.destroy(); - this.connection.emitClose(); + // Prevent close event from being emitted twice. + if (!this.closeEmitted) this.connection.emitClose(); this.debug( `[WebSocket] Manually closed the connection. | WS State: ${CONNECTION_STATE[this.connection.readyState]}`, From 043fe3e0c46a59634650c2e8f1160e8754d21356 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Thu, 10 Mar 2022 17:45:15 +0530 Subject: [PATCH 14/39] feat: add zombieConnection to typings --- typings/index.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/typings/index.d.ts b/typings/index.d.ts index 6bc7890d8173..73223083a3bd 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -4286,6 +4286,7 @@ export interface ConstantsShardEvents { INVALID_SESSION: 'invalidSession'; READY: 'ready'; RESUMED: 'resumed'; + ZOMBIE_CONNECTION: 'zombieConnection'; } export interface ConstantsStatus { From 196ec1632f4e31183a77f7cb23990bce7c77438c Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Mon, 14 Mar 2022 19:44:22 +0530 Subject: [PATCH 15/39] fix: handle edge cases with setWsCloseTimeout --- src/client/websocket/WebSocketShard.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 456f51e0aca0..ca95f5c2240f 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -559,12 +559,12 @@ class WebSocketShard extends EventEmitter { * @private */ setWsCloseTimeout(time) { + if (this.wsCloseTimeout) { + this.debug('Clearing the WebSocket close timeout.'); + clearTimeout(this.wsCloseTimeout); + } if (time === -1) { - if (this.wsCloseTimeout) { - this.debug('Clearing the WebSocket close timeout.'); - clearTimeout(this.wsCloseTimeout); - this.wsCloseTimeout = null; - } + this.wsCloseTimeout = null; return; } this.wsCloseTimeout = setTimeout(() => { @@ -814,7 +814,6 @@ class WebSocketShard extends EventEmitter { }. Forcefully terminating the connection | WS State: ${CONNECTION_STATE[this.connection.readyState]}`, ); this.connection.terminate(); - // No-op } // Emit the destroyed event if needed if (emit) this._emitDestroyed(); @@ -901,7 +900,7 @@ class WebSocketShard extends EventEmitter { * _socket.destroy() emits the close event and makes the readyState to CLOSED. */ - // manual destory + // manual destroy this.connection._socket.destroy(); // Prevent close event from being emitted twice. if (!this.closeEmitted) this.connection.emitClose(); From 60f6425c987db3f648b9ee78723ced0aac1d3417 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Wed, 20 Apr 2022 02:39:01 +0530 Subject: [PATCH 16/39] fix: connection object before accessing readyState --- src/client/websocket/WebSocketShard.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index ca95f5c2240f..2640856ace24 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -880,7 +880,11 @@ class WebSocketShard extends EventEmitter { // Wait for 400ms for ws to close. await sleep(400); - this.debug(`[WebSocket] Final Close Check. | WS State: ${CONNECTION_STATE[this.connection.readyState]}`); + this.debug( + `[WebSocket] Final Close Check. | WS State: ${ + CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] + }`, + ); if (this.connection.readyState === WebSocket.CLOSING || this.connection.readyState === WebSocket.OPEN) { this.debug( // eslint-disable-next-line max-len From 1856ac28504aad1a690cd5553852601baeb939b4 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Thu, 12 May 2022 22:55:39 +0530 Subject: [PATCH 17/39] feat(websocketshard): better way of handling zombie connection --- src/client/websocket/WebSocketManager.js | 2 +- src/client/websocket/WebSocketShard.js | 161 +++++++---------------- src/util/Constants.js | 1 + src/util/Options.js | 3 + typings/index.d.ts | 8 +- 5 files changed, 59 insertions(+), 116 deletions(-) diff --git a/src/client/websocket/WebSocketManager.js b/src/client/websocket/WebSocketManager.js index 65a23f96e338..fa2eb5ca99d9 100644 --- a/src/client/websocket/WebSocketManager.js +++ b/src/client/websocket/WebSocketManager.js @@ -20,7 +20,7 @@ const BeforeReadyWhitelist = [ WSEvents.GUILD_MEMBER_REMOVE, ]; -const UNRECOVERABLE_CLOSE_CODES = Object.keys(WSCodes).slice(1).map(Number); +const UNRECOVERABLE_CLOSE_CODES = Object.keys(WSCodes).slice(2).map(Number); const UNRESUMABLE_CLOSE_CODES = [ RPCErrorCodes.UnknownError, RPCErrorCodes.InvalidPermissions, diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 2640856ace24..f6fa4a184321 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -2,9 +2,8 @@ const EventEmitter = require('node:events'); const { setTimeout, setInterval, clearTimeout } = require('node:timers'); -const { setTimeout: sleep } = require('node:timers/promises'); const WebSocket = require('../../WebSocket'); -const { Status, Events, ShardEvents, Opcodes, WSEvents } = require('../../util/Constants'); +const { Status, Events, ShardEvents, Opcodes, WSEvents, WSCodes } = require('../../util/Constants'); const Intents = require('../../util/Intents'); const STATUS_KEYS = Object.keys(Status); @@ -204,7 +203,6 @@ class WebSocketShard extends EventEmitter { this.removeListener(ShardEvents.RESUMED, onResumed); this.removeListener(ShardEvents.INVALID_SESSION, onInvalidOrDestroyed); this.removeListener(ShardEvents.DESTROYED, onInvalidOrDestroyed); - this.removeListener(ShardEvents.ZOMBIE_CONNECTION, this.handleZombieConnection); }; const onReady = () => { @@ -358,32 +356,20 @@ class WebSocketShard extends EventEmitter { * @private */ onClose(event) { + this.closeEmitted = true; if (this.sequence !== -1) this.closeSequence = this.sequence; this.sequence = -1; - this.debug(`[CLOSE] Event Code: ${event.code} Clean : ${event.wasClean} Reason : ${event.reason ?? 'No reason received'}`); - this.setHeartbeatTimer(-1); this.setHelloTimeout(-1); - // If we still have a connection object, clean up its listeners - if (this.connection) this._cleanupConnection(); - - this.closeEmitted = true; // Clearing the WebSocket close timeout as close was emitted. this.setWsCloseTimeout(-1); - - // Step 1: Null the connection object - this.connection = null; - - // Step 2: Set the shard status to DISCONNECTED + // If we still have a connection object, clean up its listeners + if (this.connection) this._cleanupConnection(); this.status = Status.DISCONNECTED; - - // Step 3: Cache the old sequence (use to attempt a resume) - if (this.sequence !== -1) this.closeSequence = this.sequence; - /** * Emitted when a shard's WebSocket closes. * @private @@ -393,6 +379,27 @@ class WebSocketShard extends EventEmitter { this.emit(ShardEvents.CLOSE, event); } + /** + * Manually emit close since the ws never received the close frame. + */ + emitClose() { + /** + * Emitted when a shard's WebSocket closes. + * @private + * @event WebSocketShard#close + * @param {CloseEvent} event The received event + */ + this.emit(ShardEvents.CLOSE, { + code: 1011, + reason: WSCodes[1011], + /** + * - `wasClean` property is set to false when the WebSocket connection did not close via the close handshake. + * i.e. by not receiving a valid Close frame from the server. + */ + wasClean: false, + }); + } + /** * Called whenever a packet is received. * @param {Object} packet The received packet @@ -569,8 +576,8 @@ class WebSocketShard extends EventEmitter { } this.wsCloseTimeout = setTimeout(() => { this.debug(`[WebSocket] Close Emitted: ${this.closeEmitted}`); - // Check connection is null or if close event was emitted. - if (!this.connection || this.closeEmitted) { + // Check if close event was emitted. + if (this.closeEmitted) { this.debug( `[WebSocket] was closed. | WS State: ${ CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] @@ -583,17 +590,11 @@ class WebSocketShard extends EventEmitter { this.debug( // eslint-disable-next-line max-len - `[WebSocket] did not close properly, assuming a zombie connection. Destroying and reconnecting again. WS State: ${ - CONNECTION_STATE[this.connection.readyState] - } | Close Emitted: ${this.closeEmitted}`, + `[WebSocket] did not close properly, assuming a zombie connection.\nDestroying and reconnecting again. Close Emitted: ${this.closeEmitted}`, ); - /** - * Emitted when a shard's WebSocket is not closed in time. - * @private - * @event WebSocketShard#zombieConnection - */ - this.emit(ShardEvents.ZOMBIE_CONNECTION); - }, 6_000).unref(); + + this.emitClose(); + }, time).unref(); } /** @@ -651,7 +652,7 @@ class WebSocketShard extends EventEmitter { * @private */ ackHeartbeat() { - this.lastHeartbeatAcked = true; + this.lastHeartbeatAcked = false; const latency = Date.now() - this.lastPingTimestamp; this.debug(`Heartbeat acknowledged, latency of ${latency}ms.`); this.ping = latency; @@ -776,7 +777,6 @@ class WebSocketShard extends EventEmitter { destroy({ closeCode = 1_000, reset = false, emit = true, log = true } = {}) { // Making the variable false to check for zombie connections. this.closeEmitted = false; - if (log) { this.debug(`[DESTROY] Close Code : ${closeCode} @@ -789,7 +789,7 @@ class WebSocketShard extends EventEmitter { this.setHelloTimeout(-1); this.debug( - `[WS Destroy] Attempting to close the WebSocket. | WS State: ${ + `[WebSocket] Destroy: Attempting to close the WebSocket. | WS State: ${ CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] }`, ); @@ -823,30 +823,28 @@ class WebSocketShard extends EventEmitter { this._emitDestroyed(); } - /* - * Just to make sure the readyState is not stuck at CLOSING in case of a closeCode 4009. - * We can use this for other close codes as well but so far only 4009 is failing to close. - **/ - if (closeCode === 4009) { - this.debug( - // eslint-disable-next-line max-len - `[WebSocket] Encountered close code 4009. Adding a timeout to check the connection actually closed. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] - }`, - ); - this.setWsCloseTimeout(); - } + this.debug( + `[WebSocket] Adding a WebSocket close timeout to handle a perfect WS reconnect. + Timeout: ${this.manager.client.options.closeTimeout}ms`, + ); + this.setWsCloseTimeout(this.manager.client.options.closeTimeout); + + // Step 2: Null the connection object + this.connection = null; + + // Step 3: Set the shard status to DISCONNECTED + this.status = Status.DISCONNECTED; - // Step 2: Reset the sequence and session id if requested + // Step 4: Cache the old sequence (use to attempt a resume) + if (this.sequence !== -1) this.closeSequence = this.sequence; + + // Step 5: Reset the sequence and session id if requested if (reset) { this.sequence = -1; this.sessionId = null; } - // Step 3: Watch for zombie connection event which could be emitted when step 1 fails to close the WebSocket - this.on(ShardEvents.ZOMBIE_CONNECTION, this.handleZombieConnection); - - // Step 4: Reset the rate limit data + // Step 6: reset the rate limit data this.ratelimit.remaining = this.ratelimit.total; this.ratelimit.queue.length = 0; if (this.ratelimit.timer) { @@ -855,67 +853,6 @@ class WebSocketShard extends EventEmitter { } } - /** - * Forcefully closes the WebSocket when {@link WebSocketShard#destroy} could not close it properly. - * @private - * @returns {Promise} - */ - async handleZombieConnection() { - // Continuing from step 2 from the destroy method. - - // Step 3 Check if the connection was closed and readyState is at CLOSED. If not close it. - if (this.connection?.readyState === WebSocket.CLOSING || this.connection?.readyState === WebSocket.OPEN) { - this.debug( - `[WS Destroy] Step 3: WebSocket Still Connected: trying to terminate. | WS State: ${ - CONNECTION_STATE[this.connection.readyState] - }`, - ); - - // Trying to close WebSocket with terminate.. - try { - this.connection.terminate(); - } catch { - // No op. - } - - // Wait for 400ms for ws to close. - await sleep(400); - this.debug( - `[WebSocket] Final Close Check. | WS State: ${ - CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] - }`, - ); - if (this.connection.readyState === WebSocket.CLOSING || this.connection.readyState === WebSocket.OPEN) { - this.debug( - // eslint-disable-next-line max-len - `[WebSocket] Connection still stuck at Closing: calling a manual destroy of the socket to close and reconnect. | WS State: ${ - CONNECTION_STATE[this.connection.readyState] - }`, - ); - - /* - * This is an important step to deal with zombie connections wherein shards never reconnect - * after a 4009 closeCode due to the WebSocket being stuck at the CLOSING ready state. - * Check the issue https://github.com/discordjs/discord.js/issues/7450 - * - * The _socket.destroy() method ensures that no more I/O activity happens on this socket. - * Destroys the stream and closes the connection. Refer: https://nodejs.org/api/net.html#socketdestroy - * _socket.destroy() is also being invoked in ws.terminate() method. - * _socket.destroy() emits the close event and makes the readyState to CLOSED. - */ - - // manual destroy - this.connection._socket.destroy(); - // Prevent close event from being emitted twice. - if (!this.closeEmitted) this.connection.emitClose(); - - this.debug( - `[WebSocket] Manually closed the connection. | WS State: ${CONNECTION_STATE[this.connection.readyState]}`, - ); - } - } - } - /** * Cleans up the WebSocket connection listeners. * @private diff --git a/src/util/Constants.js b/src/util/Constants.js index bce640ba5a38..2ac5b6df908e 100644 --- a/src/util/Constants.js +++ b/src/util/Constants.js @@ -8,6 +8,7 @@ exports.UserAgent = `DiscordBot (${Package.homepage}, ${Package.version}) Node.j exports.WSCodes = { 1000: 'WS_CLOSE_REQUESTED', + 1011: 'INTERNAL_ERROR', 4004: 'TOKEN_INVALID', 4010: 'SHARDING_INVALID', 4011: 'SHARDING_REQUIRED', diff --git a/src/util/Options.js b/src/util/Options.js index 9b8e6bff5c2b..b02f331ea1df 100644 --- a/src/util/Options.js +++ b/src/util/Options.js @@ -33,6 +33,8 @@ const process = require('node:process'); * @property {number|number[]|string} [shards] The shard's id to run, or an array of shard ids. If not specified, * the client will spawn {@link ClientOptions#shardCount} shards. If set to `auto`, it will fetch the * recommended amount of shards from Discord and spawn that amount + * @property {number} [closeTimeout=1] The amount of time after which the shard will reconnect incase of a zombie + * connection. * @property {number} [shardCount=1] The total amount of shards used by all processes of this bot * (e.g. recommended shard count, shard count of the ShardingManager) * @property {CacheFactory} [makeCache] Function to create a cache. @@ -132,6 +134,7 @@ class Options extends null { */ static createDefault() { return { + closeTimeout: 5_000, waitGuildTimeout: 15_000, shardCount: 1, makeCache: this.cacheWithLimits(this.defaultMakeCacheSettings), diff --git a/typings/index.d.ts b/typings/index.d.ts index 51712004fd12..e01338bbe0be 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -2778,7 +2778,6 @@ export interface WebSocketShardEvents { invalidSession: []; close: [event: CloseEvent]; allReady: [unavailableGuilds?: Set]; - zombieConnection: []; } export class WebSocketShard extends EventEmitter { @@ -2821,7 +2820,7 @@ export class WebSocketShard extends EventEmitter { private _send(data: unknown): void; private processQueue(): void; private destroy(destroyOptions?: { closeCode?: number; reset?: boolean; emit?: boolean; log?: boolean }): void; - private handleZombieConnection(): Promise; + private emitClose(): void; private _cleanupConnection(): void; private _emitDestroyed(): void; @@ -2973,9 +2972,12 @@ export const Constants: { }; WSCodes: { 1000: 'WS_CLOSE_REQUESTED'; + 1011: 'INTERNAL_ERROR'; 4004: 'TOKEN_INVALID'; 4010: 'SHARDING_INVALID'; 4011: 'SHARDING_REQUIRED'; + 4013: 'INVALID_INTENTS'; + 4014: 'DISALLOWED_INTENTS'; }; Events: ConstantsEvents; ShardEvents: ConstantsShardEvents; @@ -4228,6 +4230,7 @@ export interface ClientFetchInviteOptions { export interface ClientOptions { shards?: number | number[] | 'auto'; shardCount?: number; + closeTimeout?: number; makeCache?: CacheFactory; /** @deprecated Pass the value of this property as `lifetime` to `sweepers.messages` instead. */ messageCacheLifetime?: number; @@ -4487,7 +4490,6 @@ export interface ConstantsShardEvents { INVALID_SESSION: 'invalidSession'; READY: 'ready'; RESUMED: 'resumed'; - ZOMBIE_CONNECTION: 'zombieConnection'; } export interface ConstantsStatus { From 170edfa2c340397e3b5267eb494c984a2b919567 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Thu, 12 May 2022 22:58:52 +0530 Subject: [PATCH 18/39] fix: ackHeartbeat which was used for testing purpose --- src/client/websocket/WebSocketShard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index f6fa4a184321..77a8b26ac064 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -652,7 +652,7 @@ class WebSocketShard extends EventEmitter { * @private */ ackHeartbeat() { - this.lastHeartbeatAcked = false; + this.lastHeartbeatAcked = true; const latency = Date.now() - this.lastPingTimestamp; this.debug(`Heartbeat acknowledged, latency of ${latency}ms.`); this.ping = latency; From d4cbd64fe045fecc496546c82f1f289dd6da99dc Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Thu, 12 May 2022 23:14:27 +0530 Subject: [PATCH 19/39] fix: docs for websocketshard --- src/client/websocket/WebSocketShard.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 77a8b26ac064..c8b5980047b1 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -386,7 +386,6 @@ class WebSocketShard extends EventEmitter { /** * Emitted when a shard's WebSocket closes. * @private - * @event WebSocketShard#close * @param {CloseEvent} event The received event */ this.emit(ShardEvents.CLOSE, { From c05a06244d76b170caadb160585479498cdf3799 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Thu, 12 May 2022 23:36:56 +0530 Subject: [PATCH 20/39] refactor: apply suggestions --- src/client/websocket/WebSocketShard.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index c8b5980047b1..4837c274d7b3 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -376,27 +376,27 @@ class WebSocketShard extends EventEmitter { * @event WebSocketShard#close * @param {CloseEvent} event The received event */ - this.emit(ShardEvents.CLOSE, event); + this.emitClose(event); } /** * Manually emit close since the ws never received the close frame. + * @param {CloseEvent} [event] Close event that was received */ - emitClose() { + emitClose( + event = { + code: 1011, + reason: WSCodes[1011], + wasClean: false, + }, + ) { /** * Emitted when a shard's WebSocket closes. * @private + * @event WebSocketShard#close * @param {CloseEvent} event The received event */ - this.emit(ShardEvents.CLOSE, { - code: 1011, - reason: WSCodes[1011], - /** - * - `wasClean` property is set to false when the WebSocket connection did not close via the close handshake. - * i.e. by not receiving a valid Close frame from the server. - */ - wasClean: false, - }); + this.emit(ShardEvents.CLOSE, event); } /** From c023b1879bd3cb08f230b0bf32ad994cf72445a4 Mon Sep 17 00:00:00 2001 From: Voxelli <69213593+legendhimslef@users.noreply.github.com> Date: Thu, 12 May 2022 23:44:33 +0530 Subject: [PATCH 21/39] Update src/client/websocket/WebSocketShard.js Co-authored-by: Parbez --- src/client/websocket/WebSocketShard.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 4837c274d7b3..d5323f37f9eb 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -370,12 +370,6 @@ class WebSocketShard extends EventEmitter { // If we still have a connection object, clean up its listeners if (this.connection) this._cleanupConnection(); this.status = Status.DISCONNECTED; - /** - * Emitted when a shard's WebSocket closes. - * @private - * @event WebSocketShard#close - * @param {CloseEvent} event The received event - */ this.emitClose(event); } From 37759f0954c8d306d2c8a94e9e73e7621c5dd0a9 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Thu, 12 May 2022 23:48:22 +0530 Subject: [PATCH 22/39] docs: websocketshard --- typings/index.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typings/index.d.ts b/typings/index.d.ts index e01338bbe0be..344986937f7d 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -2820,7 +2820,7 @@ export class WebSocketShard extends EventEmitter { private _send(data: unknown): void; private processQueue(): void; private destroy(destroyOptions?: { closeCode?: number; reset?: boolean; emit?: boolean; log?: boolean }): void; - private emitClose(): void; + private emitClose(event?: CloseEvent): void; private _cleanupConnection(): void; private _emitDestroyed(): void; From 8dbb5b04ba778a0951a74fb3e16afacef2d41fc8 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Fri, 13 May 2022 01:04:09 +0530 Subject: [PATCH 23/39] fix: deal with edge cases --- src/client/websocket/WebSocketShard.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index d5323f37f9eb..aec0a18fc6af 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -375,6 +375,7 @@ class WebSocketShard extends EventEmitter { /** * Manually emit close since the ws never received the close frame. + * This method helps the shard reconnect. * @param {CloseEvent} [event] Close event that was received */ emitClose( @@ -576,6 +577,7 @@ class WebSocketShard extends EventEmitter { CONNECTION_STATE[this.connection?.readyState ?? WebSocket.CLOSED] } | Close Emitted: ${this.closeEmitted}`, ); + // Setting the variable false to check for zombie connections. this.closeEmitted = false; this.setWsCloseTimeout(-1); return; @@ -587,6 +589,8 @@ class WebSocketShard extends EventEmitter { ); this.emitClose(); + // Setting the variable false to check for zombie connections. + this.closeEmitted = false; }, time).unref(); } @@ -768,8 +772,6 @@ class WebSocketShard extends EventEmitter { * @private */ destroy({ closeCode = 1_000, reset = false, emit = true, log = true } = {}) { - // Making the variable false to check for zombie connections. - this.closeEmitted = false; if (log) { this.debug(`[DESTROY] Close Code : ${closeCode} From 836d934d36617e16a8b86b234a193e453cb347a3 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Fri, 13 May 2022 02:44:19 +0530 Subject: [PATCH 24/39] refactor(websocketshard): move close debug log into emitClose method --- src/client/websocket/WebSocketShard.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index aec0a18fc6af..bfe156c56e68 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -359,10 +359,6 @@ class WebSocketShard extends EventEmitter { this.closeEmitted = true; if (this.sequence !== -1) this.closeSequence = this.sequence; this.sequence = -1; - this.debug(`[CLOSE] - Event Code: ${event.code} - Clean : ${event.wasClean} - Reason : ${event.reason ?? 'No reason received'}`); this.setHeartbeatTimer(-1); this.setHelloTimeout(-1); // Clearing the WebSocket close timeout as close was emitted. @@ -385,6 +381,10 @@ class WebSocketShard extends EventEmitter { wasClean: false, }, ) { + this.debug(`[CLOSE] + Event Code: ${event.code} + Clean : ${event.wasClean} + Reason : ${event.reason ?? 'No reason received'}`); /** * Emitted when a shard's WebSocket closes. * @private From 2f4a7e6a0d392aed61f7ae09eb5d7fba57d660ea Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Fri, 13 May 2022 03:12:25 +0530 Subject: [PATCH 25/39] fix(websocketshard): reset session id on 4009 --- src/client/websocket/WebSocketShard.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index bfe156c56e68..e9d3ad5f5df7 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -549,7 +549,7 @@ class WebSocketShard extends EventEmitter { this.debug('Setting a HELLO timeout for 20s.'); this.helloTimeout = setTimeout(() => { this.debug('Did not receive HELLO in time. Destroying and connecting again.'); - this.destroy({ closeCode: 4009 }); + this.destroy({ reset: true, closeCode: 4009 }); }, 20_000).unref(); } @@ -634,7 +634,7 @@ class WebSocketShard extends EventEmitter { Sequence : ${this.sequence} Connection State: ${this.connection ? CONNECTION_STATE[this.connection.readyState] : 'No Connection??'}`, ); - this.destroy({ closeCode: 4009 }); + this.destroy({ reset: true, closeCode: 4009 }); return; } From f0770f8778aead01d762b6e4642b476e1ad6b104 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Fri, 13 May 2022 15:12:34 +0530 Subject: [PATCH 26/39] fix: reconnect loops --- src/client/websocket/WebSocketManager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/websocket/WebSocketManager.js b/src/client/websocket/WebSocketManager.js index fa2eb5ca99d9..d8f73d1f939b 100644 --- a/src/client/websocket/WebSocketManager.js +++ b/src/client/websocket/WebSocketManager.js @@ -220,7 +220,7 @@ class WebSocketManager extends EventEmitter { this.debug(`Session id is present, attempting an immediate reconnect...`, shard); this.reconnect(); } else { - shard.destroy({ reset: true, emit: false, log: false }); + // Shard.destroy({ reset: true, emit: false, log: false }); this.reconnect(); } }); From 9b5f5f6f246caa5f97f01f1af2d61d994c839ae6 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Fri, 13 May 2022 19:15:11 +0530 Subject: [PATCH 27/39] fix: add timeout only on closing states --- src/client/websocket/WebSocketShard.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index e9d3ad5f5df7..e64362fb5d5a 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -818,11 +818,13 @@ class WebSocketShard extends EventEmitter { this._emitDestroyed(); } - this.debug( - `[WebSocket] Adding a WebSocket close timeout to handle a perfect WS reconnect. - Timeout: ${this.manager.client.options.closeTimeout}ms`, - ); - this.setWsCloseTimeout(this.manager.client.options.closeTimeout); + if (this.connection.readyState === WebSocket.CLOSING || this.connection.readyState === WebSocket.CLOSED) { + this.debug( + `[WebSocket] Adding a WebSocket close timeout to handle a perfect WS reconnect. + Timeout: ${this.manager.client.options.closeTimeout}ms`, + ); + this.setWsCloseTimeout(this.manager.client.options.closeTimeout); + } // Step 2: Null the connection object this.connection = null; From 97a2e5f9920eb26a17e6ad6277e0100c25288743 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Sat, 14 May 2022 00:40:26 +0530 Subject: [PATCH 28/39] fix: closeEmitted variable --- src/client/websocket/WebSocketShard.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index e64362fb5d5a..f6d6599ed70c 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -819,6 +819,7 @@ class WebSocketShard extends EventEmitter { } if (this.connection.readyState === WebSocket.CLOSING || this.connection.readyState === WebSocket.CLOSED) { + this.closeEmitted = false; this.debug( `[WebSocket] Adding a WebSocket close timeout to handle a perfect WS reconnect. Timeout: ${this.manager.client.options.closeTimeout}ms`, From 7194f49dec3a81d5a2374d1dccd89443d8b7896c Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Sat, 14 May 2022 18:53:35 +0530 Subject: [PATCH 29/39] fix: debug logs --- src/client/websocket/WebSocketShard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index f6d6599ed70c..8f65b385dd9d 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -585,7 +585,7 @@ class WebSocketShard extends EventEmitter { this.debug( // eslint-disable-next-line max-len - `[WebSocket] did not close properly, assuming a zombie connection.\nDestroying and reconnecting again. Close Emitted: ${this.closeEmitted}`, + `[WebSocket] did not close properly, assuming a zombie connection.\nEmitting close and reconnecting again.`, ); this.emitClose(); From f74985ac7f663adeebba5142568e34a2703f7400 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Sat, 14 May 2022 19:03:09 +0530 Subject: [PATCH 30/39] chore: cleanup --- src/client/websocket/WebSocketManager.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/websocket/WebSocketManager.js b/src/client/websocket/WebSocketManager.js index d8f73d1f939b..21a51ba4d0f0 100644 --- a/src/client/websocket/WebSocketManager.js +++ b/src/client/websocket/WebSocketManager.js @@ -220,7 +220,6 @@ class WebSocketManager extends EventEmitter { this.debug(`Session id is present, attempting an immediate reconnect...`, shard); this.reconnect(); } else { - // Shard.destroy({ reset: true, emit: false, log: false }); this.reconnect(); } }); From 745389562d7147fef560fd1a27656698d74009d7 Mon Sep 17 00:00:00 2001 From: Voxelli <69213593+legendhimslef@users.noreply.github.com> Date: Sat, 14 May 2022 19:31:03 +0530 Subject: [PATCH 31/39] chore: debug logs Co-authored-by: Vlad Frangu --- src/client/websocket/WebSocketShard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 8f65b385dd9d..f12c63319af0 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -821,7 +821,7 @@ class WebSocketShard extends EventEmitter { if (this.connection.readyState === WebSocket.CLOSING || this.connection.readyState === WebSocket.CLOSED) { this.closeEmitted = false; this.debug( - `[WebSocket] Adding a WebSocket close timeout to handle a perfect WS reconnect. + `[WebSocket] Adding a WebSocket close timeout to ensure a correct WS reconnect. Timeout: ${this.manager.client.options.closeTimeout}ms`, ); this.setWsCloseTimeout(this.manager.client.options.closeTimeout); From a3b94b2474d17fc4d8111718373c0edb0433a9bb Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Sat, 14 May 2022 19:40:33 +0530 Subject: [PATCH 32/39] refactor: remove else statement --- src/client/websocket/WebSocketManager.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/client/websocket/WebSocketManager.js b/src/client/websocket/WebSocketManager.js index 21a51ba4d0f0..8838489cc110 100644 --- a/src/client/websocket/WebSocketManager.js +++ b/src/client/websocket/WebSocketManager.js @@ -216,12 +216,8 @@ class WebSocketManager extends EventEmitter { this.shardQueue.add(shard); - if (shard.sessionId) { - this.debug(`Session id is present, attempting an immediate reconnect...`, shard); - this.reconnect(); - } else { - this.reconnect(); - } + if (shard.sessionId) this.debug(`Session id is present, attempting an immediate reconnect...`, shard); + this.reconnect(); }); shard.on(ShardEvents.INVALID_SESSION, () => { From 4a9e393d7d4b29cd8051e2eab1bce8dd18a8a121 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Mon, 16 May 2022 01:50:30 +0530 Subject: [PATCH 33/39] fix: debug logs & clearTimeout at start --- src/client/websocket/WebSocketShard.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index f12c63319af0..42571ba52b29 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -561,7 +561,7 @@ class WebSocketShard extends EventEmitter { */ setWsCloseTimeout(time) { if (this.wsCloseTimeout) { - this.debug('Clearing the WebSocket close timeout.'); + this.debug('[WebSocket] Clearing the close timeout.'); clearTimeout(this.wsCloseTimeout); } if (time === -1) { @@ -569,6 +569,7 @@ class WebSocketShard extends EventEmitter { return; } this.wsCloseTimeout = setTimeout(() => { + this.setWsCloseTimeout(-1); this.debug(`[WebSocket] Close Emitted: ${this.closeEmitted}`); // Check if close event was emitted. if (this.closeEmitted) { @@ -579,7 +580,6 @@ class WebSocketShard extends EventEmitter { ); // Setting the variable false to check for zombie connections. this.closeEmitted = false; - this.setWsCloseTimeout(-1); return; } From 0daf30450efc66e287fc1454b377a2f7bca50d4d Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Mon, 16 May 2022 14:06:21 +0530 Subject: [PATCH 34/39] fix: docs --- src/util/Options.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/Options.js b/src/util/Options.js index b02f331ea1df..7034c439d9c0 100644 --- a/src/util/Options.js +++ b/src/util/Options.js @@ -33,8 +33,8 @@ const process = require('node:process'); * @property {number|number[]|string} [shards] The shard's id to run, or an array of shard ids. If not specified, * the client will spawn {@link ClientOptions#shardCount} shards. If set to `auto`, it will fetch the * recommended amount of shards from Discord and spawn that amount - * @property {number} [closeTimeout=1] The amount of time after which the shard will reconnect incase of a zombie - * connection. + * @property {number} [closeTimeout=1] The amount of time in milliseconds after which the shard will reconnect + * in case of a zombie connection. * @property {number} [shardCount=1] The total amount of shards used by all processes of this bot * (e.g. recommended shard count, shard count of the ShardingManager) * @property {CacheFactory} [makeCache] Function to create a cache. From 58772461b2bbe3abddda174bc1ff89ee756a1150 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Fri, 20 May 2022 18:26:52 +0530 Subject: [PATCH 35/39] docs: elaborated a bit more --- src/client/websocket/WebSocketShard.js | 2 +- src/util/Options.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index 42571ba52b29..c25d9f959305 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -370,7 +370,7 @@ class WebSocketShard extends EventEmitter { } /** - * Manually emit close since the ws never received the close frame. + * This method is responsible to emit close event for this shard. * This method helps the shard reconnect. * @param {CloseEvent} [event] Close event that was received */ diff --git a/src/util/Options.js b/src/util/Options.js index 7034c439d9c0..b66c7cbc295d 100644 --- a/src/util/Options.js +++ b/src/util/Options.js @@ -33,8 +33,8 @@ const process = require('node:process'); * @property {number|number[]|string} [shards] The shard's id to run, or an array of shard ids. If not specified, * the client will spawn {@link ClientOptions#shardCount} shards. If set to `auto`, it will fetch the * recommended amount of shards from Discord and spawn that amount - * @property {number} [closeTimeout=1] The amount of time in milliseconds after which the shard will reconnect - * in case of a zombie connection. + * @property {number} [closeTimeout=1] The amount of time in milliseconds to wait for the close frame to be received + * from the WebSocket. Don't have this too high/low. Its best to have it between 2_000-6_000 ms. * @property {number} [shardCount=1] The total amount of shards used by all processes of this bot * (e.g. recommended shard count, shard count of the ShardingManager) * @property {CacheFactory} [makeCache] Function to create a cache. From 7b108a482483ec39109c8911f5d78b6fa2a8ba43 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Fri, 20 May 2022 20:56:07 +0530 Subject: [PATCH 36/39] fix: if statement --- src/client/websocket/WebSocketShard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index c25d9f959305..a190b3987f33 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -818,7 +818,7 @@ class WebSocketShard extends EventEmitter { this._emitDestroyed(); } - if (this.connection.readyState === WebSocket.CLOSING || this.connection.readyState === WebSocket.CLOSED) { + if (this.connection?.readyState === WebSocket.CLOSING || this.connection?.readyState === WebSocket.CLOSED) { this.closeEmitted = false; this.debug( `[WebSocket] Adding a WebSocket close timeout to ensure a correct WS reconnect. From be048199a2891883554112d8c8679a4de5b866ef Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Tue, 24 May 2022 20:05:39 +0530 Subject: [PATCH 37/39] feat: handle edge cases while reconnecting --- src/client/websocket/WebSocketShard.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index a190b3987f33..f77d93d92932 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -265,7 +265,7 @@ class WebSocketShard extends EventEmitter { this.status = this.status === Status.DISCONNECTED ? Status.RECONNECTING : Status.CONNECTING; this.setHelloTimeout(); - + this.setWsCloseTimeout(-1); this.connectedAt = Date.now(); // Adding a handshake timeout to just make sure no zombie connection appears. @@ -797,8 +797,6 @@ class WebSocketShard extends EventEmitter { } else { // Connection is not OPEN this.debug(`WS State: ${CONNECTION_STATE[this.connection.readyState]}`); - // Remove listeners from the connection - this._cleanupConnection(); // Attempt to close the connection just in case try { this.connection.close(closeCode); From 88fb5f6af503e72f38853b644e8579b74df65227 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Thu, 26 May 2022 08:03:07 +0530 Subject: [PATCH 38/39] fix: deal with an edge case Invoke destroy when connection object exists in onClose method for a cleanup and correct reconnect. --- src/client/websocket/WebSocketShard.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/websocket/WebSocketShard.js b/src/client/websocket/WebSocketShard.js index f77d93d92932..43faa8ee85c7 100644 --- a/src/client/websocket/WebSocketShard.js +++ b/src/client/websocket/WebSocketShard.js @@ -364,7 +364,11 @@ class WebSocketShard extends EventEmitter { // Clearing the WebSocket close timeout as close was emitted. this.setWsCloseTimeout(-1); // If we still have a connection object, clean up its listeners - if (this.connection) this._cleanupConnection(); + if (this.connection) { + this._cleanupConnection(); + // Having this after _cleanupConnection to just clean up the connection and not listen to ws.onclose + this.destroy({ reset: true, emit: false, log: false }); + } this.status = Status.DISCONNECTED; this.emitClose(event); } From 931fac54e29e38bfcc8ba521c43af9cb0ed2acd8 Mon Sep 17 00:00:00 2001 From: legendhimslef Date: Mon, 30 May 2022 02:38:51 +0530 Subject: [PATCH 39/39] chore: cleanup --- src/util/Constants.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/util/Constants.js b/src/util/Constants.js index 2ac5b6df908e..2169c1f2f626 100644 --- a/src/util/Constants.js +++ b/src/util/Constants.js @@ -221,7 +221,6 @@ exports.ShardEvents = { READY: 'ready', RESUMED: 'resumed', ALL_READY: 'allReady', - ZOMBIE_CONNECTION: 'zombieConnection', }; /**