In lameduck mode shutdown jetstream at start, do not leave running (#…

…4579)

Signed-off-by: Derek Collison <derek@nats.io>

server/consumer.go

@@ -1564,7 +1564,7 @@ func (o *consumer) deleteNotActive() {
 			// Don't think this needs to be a monitored go routine.
 			go func() {
 				const (
-					startInterval = 5 * time.Second
+					startInterval = 30 * time.Second
 					maxInterval   = 5 * time.Minute
 				)
 				jitter := time.Duration(rand.Int63n(int64(startInterval)))

server/jetstream_cluster_3_test.go

@@ -3472,13 +3472,13 @@ func TestJetStreamClusterNoR1AssetsDuringLameDuck(t *testing.T) {
 
 	// Make sure we do not have any R1 assets placed on the lameduck server.
 	for s.isRunning() {
-		s.rnMu.RLock()
+		s.mu.RLock()
 		if s.js == nil || s.js.srv == nil || s.js.srv.gacc == nil {
-			s.rnMu.RUnlock()
+			s.mu.RUnlock()
 			break
 		}
 		hasAsset := len(s.js.srv.gacc.streams()) > 0
-		s.rnMu.RUnlock()
+		s.mu.RUnlock()
 		if hasAsset {
 			t.Fatalf("Server had an R1 asset when it should not due to lameduck mode")
 		}

server/raft.go

@@ -588,14 +588,15 @@ func (s *Server) stepdownRaftNodes() {
 		s.Debugf("Stepping down all leader raft nodes")
 	}
 	for _, n := range s.raftNodes {
-		if n.Leader() {
-			nodes = append(nodes, n)
-		}
+		nodes = append(nodes, n)
 	}
 	s.rnMu.RUnlock()
 
 	for _, node := range nodes {
-		node.StepDown()
+		if node.Leader() {
+			node.StepDown()
+		}
+		node.SetObserver(true)
 	}
 }
 

server/server.go

@@ -2380,7 +2380,7 @@ func (s *Server) Shutdown() {
 		accRes.Close()
 	}
 
-	// Now check jetstream.
+	// Now check and shutdown jetstream.
 	s.shutdownJetStream()
 
 	// Now shutdown the nodes
@@ -3944,7 +3944,8 @@ func (s *Server) isLameDuckMode() bool {
 }
 
 // This function will close the client listener then close the clients
-// at some interval to avoid a reconnecting storm.
+// at some interval to avoid a reconnect storm.
+// We will also transfer any raft leaders and shutdown JetStream.
 func (s *Server) lameDuckMode() {
 	s.mu.Lock()
 	// Check if there is actually anything to do
@@ -3985,6 +3986,12 @@ func (s *Server) lameDuckMode() {
 		}
 	}
 
+	// Now check and shutdown jetstream.
+	s.shutdownJetStream()
+
+	// Now shutdown the nodes
+	s.shutdownRaftNodes()
+
 	// Wait for accept loops to be done to make sure that no new
 	// client can connect
 	for i := 0; i < expected; i++ {