Allow a small number of timeouts to prevent flapping

Signed-off-by: Derek Collison <derek@nats.io>

test/js_test.go

@@ -6278,7 +6278,7 @@ func TestJetStreamClusterStreamLeaderChangeClientErr(t *testing.T) {
 	withJSClusterAndStream(t, "R3S", 3, cfg, func(t *testing.T, stream string, servers ...*jsServer) {
 		// We want to make sure the worse thing seen by the lower levels during a leadership change is NoResponders.
 		// We will have three concurrent contexts going on.
-		// 1. Leadership Changes every second.
+		// 1. Leadership Changes every 500ms.
 		// 2. Publishing messages to the stream every 10ms.
 		// 3. StreamInfo calls every 15ms.
 		expires := time.Now().Add(5 * time.Second)
@@ -6297,7 +6297,7 @@ func TestJetStreamClusterStreamLeaderChangeClientErr(t *testing.T) {
 
 			sds := fmt.Sprintf(server.JSApiStreamLeaderStepDownT, "TEST")
 			for time.Now().Before(expires) {
-				time.Sleep(1 * time.Second)
+				time.Sleep(500 * time.Millisecond)
 				si, err := js.StreamInfo("TEST")
 				expectOk(t, err)
 				_, err = nc.Request(sds, nil, time.Second)
@@ -6316,6 +6316,7 @@ func TestJetStreamClusterStreamLeaderChangeClientErr(t *testing.T) {
 		}()
 
 		// Published every 10ms
+		toc := 0
 		go func() {
 			defer wg.Done()
 			nc, js := jsClient(t, randServer())
@@ -6324,6 +6325,10 @@ func TestJetStreamClusterStreamLeaderChangeClientErr(t *testing.T) {
 			for time.Now().Before(expires) {
 				time.Sleep(10 * time.Millisecond)
 				_, err := js.Publish("foo", []byte("OK"))
+				if err == nats.ErrTimeout {
+					toc++
+					continue
+				}
 				expectOk(t, err)
 			}
 		}()
@@ -6342,5 +6347,10 @@ func TestJetStreamClusterStreamLeaderChangeClientErr(t *testing.T) {
 		}()
 
 		wg.Wait()
+
+		// An occasional timeout can occur, but should be 0 or maybe 1 with ~10 leadership changes per test run.
+		if toc > 1 {
+			t.Fatalf("Got too many timeout errors from publish: %d", toc)
+		}
 	})
 }