fix(SENTINEL): actively failover detection under an option (#1363)

Author: luin

lib/connectors/SentinelConnector/index.ts

@@ -52,6 +52,7 @@ export interface ISentinelConnectionOptions extends ITcpConnectionOptions {
   natMap?: INatMap;
   updateSentinels?: boolean;
   sentinelMaxConnections?: number;
+  failoverDetector?: boolean;
 }
 
 export default class SentinelConnector extends AbstractConnector {
@@ -321,6 +322,9 @@ export default class SentinelConnector extends AbstractConnector {
   }
 
   private async initFailoverDetector(): Promise<void> {
+    if (!this.options.failoverDetector) {
+      return;
+    }
     // Move the current sentinel to the first position
     this.sentinelIterator.reset(true);
 

lib/redis/RedisOptions.ts

@@ -63,6 +63,7 @@ export const DEFAULT_REDIS_OPTIONS: IRedisOptions = {
   natMap: null,
   enableTLSForSentinelMode: false,
   updateSentinels: true,
+  failoverDetector: false,
   // Status
   username: null,
   password: null,

lib/redis/index.ts

@@ -98,6 +98,11 @@ const debug = Debug("redis");
  * @param {NatMap} [options.natMap=null] NAT map for sentinel connector.
  * @param {boolean} [options.updateSentinels=true] - Update the given `sentinels` list with new IP
  * addresses when communicating with existing sentinels.
+ * @param {boolean} [options.failoverDetector=false] - Detect failover actively by subscribing to the
+ * related channels. With this option disabled, ioredis is still able to detect failovers because Redis
+ * Sentinel will disconnect all clients whenever a failover happens, so ioredis will reconnect to the new
+ * master. This option is useful when you want to detect failover quicker, but it will create more TCP
+ * connections to Redis servers in order to subscribe to related channels.
 * @param {boolean} [options.enableAutoPipelining=false] - When enabled, all commands issued during an event loop
  * iteration are automatically wrapped in a pipeline and sent to the server at the same time.
  * This can dramatically improve performance.

test/functional/sentinel.ts

@@ -556,242 +556,248 @@ describe("sentinel", function () {
       });
     });
 
-    it("should connect to new master after +switch-master", async function () {
-      const sentinel = new MockServer(27379, function (argv) {
-        if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
-          return ["127.0.0.1", "17380"];
-        }
-      });
-      const master = new MockServer(17380);
-      const newMaster = new MockServer(17381);
-
-      const redis = new Redis({
-        sentinels: [{ host: "127.0.0.1", port: 27379 }],
-        name: "master",
-      });
-
-      await Promise.all([
-        once(master, "connect"),
-        once(redis, "failoverSubscribed"),
-      ]);
+    describe("failoverDetector", () => {
+      it("should connect to new master after +switch-master", async function () {
+        const sentinel = new MockServer(27379, function (argv) {
+          if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
+            return ["127.0.0.1", "17380"];
+          }
+        });
+        const master = new MockServer(17380);
+        const newMaster = new MockServer(17381);
 
-      sentinel.handler = function (argv) {
-        if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
-          return ["127.0.0.1", "17381"];
-        }
-      };
+        const redis = new Redis({
+          sentinels: [{ host: "127.0.0.1", port: 27379 }],
+          failoverDetector: true,
+          name: "master",
+        });
 
-      sentinel.broadcast([
-        "message",
-        "+switch-master",
-        "master 127.0.0.1 17380 127.0.0.1 17381",
-      ]);
+        await Promise.all([
+          once(master, "connect"),
+          once(redis, "failoverSubscribed"),
+        ]);
 
-      await Promise.all([
-        once(redis, "close"), // Wait until disconnects from old master
-        once(master, "disconnect"),
-        once(newMaster, "connect"),
-      ]);
+        sentinel.handler = function (argv) {
+          if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
+            return ["127.0.0.1", "17381"];
+          }
+        };
 
-      redis.disconnect(); // Disconnect from new master
+        sentinel.broadcast([
+          "message",
+          "+switch-master",
+          "master 127.0.0.1 17380 127.0.0.1 17381",
+        ]);
 
-      await Promise.all([
-        sentinel.disconnectPromise(),
-        master.disconnectPromise(),
-        newMaster.disconnectPromise(),
-      ]);
-    });
+        await Promise.all([
+          once(redis, "close"), // Wait until disconnects from old master
+          once(master, "disconnect"),
+          once(newMaster, "connect"),
+        ]);
 
-    it("should detect failover from secondary sentinel", async function () {
-      const sentinel1 = new MockServer(27379, function (argv) {
-        if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
-          return ["127.0.0.1", "17380"];
-        }
-      });
-      const sentinel2 = new MockServer(27380);
-      const master = new MockServer(17380);
-      const newMaster = new MockServer(17381);
+        redis.disconnect(); // Disconnect from new master
 
-      const redis = new Redis({
-        sentinels: [
-          { host: "127.0.0.1", port: 27379 },
-          { host: "127.0.0.1", port: 27380 },
-        ],
-        name: "master",
+        await Promise.all([
+          sentinel.disconnectPromise(),
+          master.disconnectPromise(),
+          newMaster.disconnectPromise(),
+        ]);
       });
 
-      await Promise.all([
-        once(master, "connect"),
-        once(redis, "failoverSubscribed"),
-      ]);
+      it("should detect failover from secondary sentinel", async function () {
+        const sentinel1 = new MockServer(27379, function (argv) {
+          if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
+            return ["127.0.0.1", "17380"];
+          }
+        });
+        const sentinel2 = new MockServer(27380);
+        const master = new MockServer(17380);
+        const newMaster = new MockServer(17381);
+
+        const redis = new Redis({
+          sentinels: [
+            { host: "127.0.0.1", port: 27379 },
+            { host: "127.0.0.1", port: 27380 },
+          ],
+          name: "master",
+          failoverDetector: true,
+        });
 
-      // In this test, only the first sentinel is used to resolve the master
-      sentinel1.handler = function (argv) {
-        if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
-          return ["127.0.0.1", "17381"];
-        }
-      };
+        await Promise.all([
+          once(master, "connect"),
+          once(redis, "failoverSubscribed"),
+        ]);
 
-      // But only the second sentinel broadcasts +switch-master
-      sentinel2.broadcast([
-        "message",
-        "+switch-master",
-        "master 127.0.0.1 17380 127.0.0.1 17381",
-      ]);
+        // In this test, only the first sentinel is used to resolve the master
+        sentinel1.handler = function (argv) {
+          if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
+            return ["127.0.0.1", "17381"];
+          }
+        };
 
-      await Promise.all([
-        once(redis, "close"), // Wait until disconnects from old master
-        once(master, "disconnect"),
-        once(newMaster, "connect"),
-      ]);
+        // But only the second sentinel broadcasts +switch-master
+        sentinel2.broadcast([
+          "message",
+          "+switch-master",
+          "master 127.0.0.1 17380 127.0.0.1 17381",
+        ]);
 
-      redis.disconnect(); // Disconnect from new master
+        await Promise.all([
+          once(redis, "close"), // Wait until disconnects from old master
+          once(master, "disconnect"),
+          once(newMaster, "connect"),
+        ]);
 
-      await Promise.all([
-        sentinel1.disconnectPromise(),
-        sentinel2.disconnectPromise(),
-        master.disconnectPromise(),
-        newMaster.disconnectPromise(),
-      ]);
-    });
-
-    it("should detect failover when some sentinels fail", async function () {
-      // Will disconnect before failover
-      const sentinel1 = new MockServer(27379, function (argv) {
-        if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
-          return ["127.0.0.1", "17380"];
-        }
-      });
+        redis.disconnect(); // Disconnect from new master
 
-      // Will emit an error before failover
-      let sentinel2Socket: Socket | null = null;
-      const sentinel2 = new MockServer(27380, function (argv, socket) {
-        sentinel2Socket = socket;
+        await Promise.all([
+          sentinel1.disconnectPromise(),
+          sentinel2.disconnectPromise(),
+          master.disconnectPromise(),
+          newMaster.disconnectPromise(),
+        ]);
       });
 
-      // Fails to subscribe
-      const sentinel3 = new MockServer(27381, function (argv, socket, flags) {
-        if (argv[0] === "subscribe") {
-          triggerParseError(socket);
-        }
-      });
-
-      // The only sentinel that can successfully publish the failover message
-      const sentinel4 = new MockServer(27382);
+      it("should detect failover when some sentinels fail", async function () {
+        // Will disconnect before failover
+        const sentinel1 = new MockServer(27379, function (argv) {
+          if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
+            return ["127.0.0.1", "17380"];
+          }
+        });
 
-      const master = new MockServer(17380);
-      const newMaster = new MockServer(17381);
+        // Will emit an error before failover
+        let sentinel2Socket: Socket | null = null;
+        const sentinel2 = new MockServer(27380, function (argv, socket) {
+          sentinel2Socket = socket;
+        });
 
-      const redis = new Redis({
-        sentinels: [
-          { host: "127.0.0.1", port: 27379 },
-          { host: "127.0.0.1", port: 27380 },
-          { host: "127.0.0.1", port: 27381 },
-          { host: "127.0.0.1", port: 27382 },
-        ],
-        name: "master",
-      });
+        // Fails to subscribe
+        const sentinel3 = new MockServer(27381, function (argv, socket, flags) {
+          if (argv[0] === "subscribe") {
+            triggerParseError(socket);
+          }
+        });
 
-      await Promise.all([
-        once(master, "connect"),
+        // The only sentinel that can successfully publish the failover message
+        const sentinel4 = new MockServer(27382);
+
+        const master = new MockServer(17380);
+        const newMaster = new MockServer(17381);
+
+        const redis = new Redis({
+          sentinels: [
+            { host: "127.0.0.1", port: 27379 },
+            { host: "127.0.0.1", port: 27380 },
+            { host: "127.0.0.1", port: 27381 },
+            { host: "127.0.0.1", port: 27382 },
+          ],
+          name: "master",
+          failoverDetector: true,
+        });
 
-        // Must resolve even though subscribing to sentinel3 fails
-        once(redis, "failoverSubscribed"),
-      ]);
+        await Promise.all([
+          once(master, "connect"),
 
-      // Fail sentinels 1 and 2
-      await sentinel1.disconnectPromise();
-      triggerParseError(sentinel2Socket);
+          // Must resolve even though subscribing to sentinel3 fails
+          once(redis, "failoverSubscribed"),
+        ]);
 
-      sentinel4.handler = function (argv) {
-        if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
-          return ["127.0.0.1", "17381"];
-        }
-      };
+        // Fail sentinels 1 and 2
+        await sentinel1.disconnectPromise();
+        triggerParseError(sentinel2Socket);
 
-      sentinel4.broadcast([
-        "message",
-        "+switch-master",
-        "master 127.0.0.1 17380 127.0.0.1 17381",
-      ]);
+        sentinel4.handler = function (argv) {
+          if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
+            return ["127.0.0.1", "17381"];
+          }
+        };
 
-      await Promise.all([
-        once(redis, "close"), // Wait until disconnects from old master
-        once(master, "disconnect"),
-        once(newMaster, "connect"),
-      ]);
+        sentinel4.broadcast([
+          "message",
+          "+switch-master",
+          "master 127.0.0.1 17380 127.0.0.1 17381",
+        ]);
 
-      redis.disconnect(); // Disconnect from new master
+        await Promise.all([
+          once(redis, "close"), // Wait until disconnects from old master
+          once(master, "disconnect"),
+          once(newMaster, "connect"),
+        ]);
 
-      await Promise.all([
-        // sentinel1 is already disconnected
-        sentinel2.disconnectPromise(),
-        sentinel3.disconnectPromise(),
-        sentinel4.disconnectPromise(),
-        master.disconnectPromise(),
-        newMaster.disconnectPromise(),
-      ]);
-    });
+        redis.disconnect(); // Disconnect from new master
 
-    it("should detect failover after sentinel disconnects and reconnects", async function () {
-      const sentinel = new MockServer(27379, function (argv) {
-        if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
-          return ["127.0.0.1", "17380"];
-        }
+        await Promise.all([
+          // sentinel1 is already disconnected
+          sentinel2.disconnectPromise(),
+          sentinel3.disconnectPromise(),
+          sentinel4.disconnectPromise(),
+          master.disconnectPromise(),
+          newMaster.disconnectPromise(),
+        ]);
       });
 
-      const master = new MockServer(17380);
-      const newMaster = new MockServer(17381);
+      it("should detect failover after sentinel disconnects and reconnects", async function () {
+        const sentinel = new MockServer(27379, function (argv) {
+          if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
+            return ["127.0.0.1", "17380"];
+          }
+        });
 
-      const redis = new Redis({
-        sentinels: [{ host: "127.0.0.1", port: 27379 }],
-        name: "master",
-        sentinelReconnectStrategy: () => 1000,
-      });
+        const master = new MockServer(17380);
+        const newMaster = new MockServer(17381);
 
-      await Promise.all([
-        once(master, "connect"),
-        once(redis, "failoverSubscribed"),
-      ]);
+        const redis = new Redis({
+          sentinels: [{ host: "127.0.0.1", port: 27379 }],
+          name: "master",
+          sentinelReconnectStrategy: () => 1000,
+          failoverDetector: true,
+        });
 
-      await sentinel.disconnectPromise();
+        await Promise.all([
+          once(master, "connect"),
+          once(redis, "failoverSubscribed"),
+        ]);
 
-      sentinel.handler = function (argv) {
-        if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
-          return ["127.0.0.1", "17381"];
-        }
-        if (argv[0] === "subscribe") {
-          sentinel.emit("test:resubscribed"); // Custom event only used in tests
-        }
-      };
+        await sentinel.disconnectPromise();
 
-      sentinel.connect();
+        sentinel.handler = function (argv) {
+          if (argv[0] === "sentinel" && argv[1] === "get-master-addr-by-name") {
+            return ["127.0.0.1", "17381"];
+          }
+          if (argv[0] === "subscribe") {
+            sentinel.emit("test:resubscribed"); // Custom event only used in tests
+          }
+        };
 
-      const clock = sinon.useFakeTimers();
-      await once(redis, "sentinelReconnecting"); // Wait for the timeout to be set
-      clock.tick(1000);
-      clock.restore();
-      await once(sentinel, "test:resubscribed");
+        sentinel.connect();
 
-      sentinel.broadcast([
-        "message",
-        "+switch-master",
-        "master 127.0.0.1 17380 127.0.0.1 17381",
-      ]);
+        const clock = sinon.useFakeTimers();
+        await once(redis, "sentinelReconnecting"); // Wait for the timeout to be set
+        clock.tick(1000);
+        clock.restore();
+        await once(sentinel, "test:resubscribed");
 
-      await Promise.all([
-        once(redis, "close"), // Wait until disconnects from old master
-        once(master, "disconnect"),
-        once(newMaster, "connect"),
-      ]);
+        sentinel.broadcast([
+          "message",
+          "+switch-master",
+          "master 127.0.0.1 17380 127.0.0.1 17381",
+        ]);
 
-      redis.disconnect(); // Disconnect from new master
+        await Promise.all([
+          once(redis, "close"), // Wait until disconnects from old master
+          once(master, "disconnect"),
+          once(newMaster, "connect"),
+        ]);
 
-      await Promise.all([
-        sentinel.disconnectPromise(),
-        master.disconnectPromise(),
-        newMaster.disconnectPromise(),
-      ]);
+        redis.disconnect(); // Disconnect from new master
+
+        await Promise.all([
+          sentinel.disconnectPromise(),
+          master.disconnectPromise(),
+          newMaster.disconnectPromise(),
+        ]);
+      });
     });
   });
 });