Add randomness in robustness cluster process version to test mixed ve…

…rsion scenarios.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

tests/framework/e2e/cluster.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"math/rand"
 	"net/url"
 	"path"
 	"path/filepath"
@@ -31,6 +32,7 @@ import (
 	"go.uber.org/zap/zaptest"
 
 	"go.etcd.io/etcd/api/v3/etcdserverpb"
+	"go.etcd.io/etcd/client/pkg/v3/fileutil"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/proxy"
 	"go.etcd.io/etcd/server/v3/embed"
@@ -148,6 +150,9 @@ type EtcdProcessClusterConfig struct {
 
 	EnvVars map[string]string
 	Version ClusterVersion
+	// LastReleaseProbability is the probability of a process with the last release binary,
+	// used in combination with Version = RandomVersion.
+	LastReleaseProbability float64
 
 	// Cluster setup config
 
@@ -207,6 +212,10 @@ func WithVersion(version ClusterVersion) EPClusterOption {
 	return func(c *EtcdProcessClusterConfig) { c.Version = version }
 }
 
+func WithLastReleaseProbability(prob float64) EPClusterOption {
+	return func(c *EtcdProcessClusterConfig) { c.LastReleaseProbability = prob }
+}
+
 func WithDataDirPath(path string) EPClusterOption {
 	return func(c *EtcdProcessClusterConfig) { c.BaseDataDirPath = path }
 }
@@ -609,6 +618,8 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 		}
 	case LastVersion:
 		execPath = BinPath.EtcdLastRelease
+	case RandomVersion:
+		execPath = pickRandomVersion(cfg.Logger, cfg.LastReleaseProbability)
 	default:
 		panic(fmt.Sprintf("Unknown cluster version %v", cfg.Version))
 	}
@@ -635,6 +646,23 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 	}
 }
 
+func pickRandomVersion(lg *zap.Logger, lastReleaseProbability float64) string {
+	execPath := BinPath.Etcd
+	if !fileutil.Exist(BinPath.EtcdLastRelease) {
+		lg.Warn("EtcdLastRelease needs to exist to use Random version. Falling back to CurrentVersion", zap.String("EtcdLastRelease-path", BinPath.EtcdLastRelease))
+		return execPath
+	}
+	if lastReleaseProbability < 0 || lastReleaseProbability > 1 {
+		panic(fmt.Sprintf("LastReleaseProbability has to be between [0, 1.0], got %v", lastReleaseProbability))
+	}
+	r := rand.Float64()
+	if r < lastReleaseProbability {
+		execPath = BinPath.EtcdLastRelease
+	}
+	lg.Info("picked random process binary path", zap.String("exec-path", execPath), zap.Float64("LastReleaseProbability", lastReleaseProbability))
+	return execPath
+}
+
 func values(cfg embed.Config) map[string]string {
 	fs := flag.NewFlagSet("etcd", flag.ContinueOnError)
 	cfg.AddFlags(fs)

tests/framework/e2e/config.go

@@ -21,6 +21,7 @@ const (
 	MinorityLastVersion ClusterVersion = "minority-last-version"
 	QuorumLastVersion   ClusterVersion = "quorum-last-version"
 	LastVersion         ClusterVersion = "last-version"
+	RandomVersion       ClusterVersion = "random-version"
 )
 
 type ClusterContext struct {

tests/robustness/scenarios.go

@@ -105,6 +105,9 @@ func exploratoryScenarios(t *testing.T) []testScenario {
 		if !v.LessThan(version.V3_6) {
 			clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithSnapshotCatchUpEntries(100))
 		}
+		if !v.LessThan(version.V3_5) {
+			clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithVersion(e2e.RandomVersion), e2e.WithLastReleaseProbability(0.1))
+		}
 		scenarios = append(scenarios, testScenario{
 			name:    name,
 			traffic: tp.Traffic,