.github/workflows/gh-workflow-approve.yaml

@@ -0,0 +1,40 @@
+---
+name: Approve GitHub Workflows
+
+on:
+  pull_request_target:
+    types:
+      - labeled
+      - synchronize
+
+jobs:
+  approve:
+    name: Approve ok-to-test
+    if: contains(github.event.pull_request.labels.*.name, 'ok-to-test')
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    steps:
+      - name: Update PR
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        continue-on-error: true
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          debug: ${{ secrets.ACTIONS_RUNNER_DEBUG == 'true' }}
+          script: |
+            const result = await github.rest.actions.listWorkflowRunsForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              event: "pull_request",
+              status: "action_required",
+              head_sha: context.payload.pull_request.head.sha,
+              per_page: 100
+            });
+
+            for (var run of result.data.workflow_runs) {
+              await github.rest.actions.approveWorkflowRun({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: run.id
+              });
+            }

.github/workflows/release.yaml

@@ -4,10 +4,10 @@ jobs:
   main:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - id: goversion
       run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
-    - uses: actions/setup-go@v2
+    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
       with:
         go-version: ${{ steps.goversion.outputs.goversion }}
     - name: release
@@ -30,3 +30,41 @@ jobs:
     - name: test-image
       run: |
         VERSION=3.5.99 ./scripts/test_images.sh
+    - name: save-image
+      run: |
+        docker image save -o /tmp/etcd-img.tar gcr.io/etcd-development/etcd
+    - name: upload-image
+      uses: actions/upload-artifact@v3
+      with:
+        name: etcd-img
+        path: /tmp/etcd-img.tar
+        retention-days: 1
+  trivy-scan:
+    needs: main
+    strategy:
+      fail-fast: false
+      matrix:
+        platforms: [amd64, arm64, ppc64le, s390x]
+    permissions:
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+    runs-on: ubuntu-latest
+    steps:
+      - name: get-image
+        uses: actions/download-artifact@v3
+        with:
+          name: etcd-img
+          path: /tmp
+      - name: load-image
+        run: |
+          docker load < /tmp/etcd-img.tar
+      - name: trivy-scan
+        uses: aquasecurity/trivy-action@d710430a6722f083d3b36b8339ff66b32f22ee55 # v0.19.0
+        with:
+          image-ref: 'gcr.io/etcd-development/etcd:v3.5.99-${{ matrix.platforms }}'
+          severity: 'CRITICAL,HIGH'
+          format: 'sarif'
+          output: 'trivy-results-${{ matrix.platforms }}.sarif'
+      - name: upload scan results
+        uses: github/codeql-action/upload-sarif@956f09c2ef1926b580554b9014cfb8a51abf89dd # v2.16.6
+        with:
+          sarif_file: 'trivy-results-${{ matrix.platforms }}.sarif'

.github/workflows/static-analysis.yaml

@@ -0,0 +1,32 @@
+---
+name: Static Analysis
+on: [push, pull_request]
+permissions: read-all
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+      - id: goversion
+        run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
+      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+        with:
+          go-version: ${{ steps.goversion.outputs.goversion }}
+      - run: |
+          set -euo pipefail
+
+          make verify
+      - run: |
+          set -euo pipefail
+
+          make fix
+
+          DIFF=$(git status --porcelain)
+
+          if [ -n "$DIFF" ]; then
+            echo "These files were modified:"
+            echo
+            echo "$DIFF"
+            echo
+            exit 1
+          fi

.github/workflows/tests-template.yaml

@@ -41,13 +41,16 @@ jobs:
               GOARCH=${{ inputs.arch }} CPU=4 RACE='false' make test-smoke
               ;;
             linux-integration-1-cpu)
-              GOARCH=${{ inputs.arch }} CPU=1 RACE='false' make test-integration
+              make install-gofail
+              GOARCH=${{ inputs.arch }} CPU=1 RACE='false' FAILPOINTS='true' make test-integration
               ;;
             linux-integration-2-cpu)
-              GOARCH=${{ inputs.arch }} CPU=2 RACE='false' make test-integration
+              make install-gofail
+              GOARCH=${{ inputs.arch }} CPU=2 RACE='false' FAILPOINTS='true' make test-integration
               ;;
             linux-integration-4-cpu)
-              GOARCH=${{ inputs.arch }} CPU=4 RACE='false' make test-integration
+              make install-gofail
+              GOARCH=${{ inputs.arch }} CPU=4 RACE='false' FAILPOINTS='true' make test-integration
               ;;
             linux-unit-4-cpu-race)
               GOARCH=${{ inputs.arch }} RACE='true' CPU='4' GO_TEST_FLAGS='-p=2' make test-unit

.go-version

@@ -1 +1 @@
-1.21.8
+1.21.10

Dockerfile-release.amd64

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 gcr.io/distroless/static-debian11@sha256:7e5c6a2a4ae854242874d36171b31d26e0539c98fc6080f942f16b03e82851ab
+FROM --platform=linux/amd64 gcr.io/distroless/static-debian11@sha256:6d31326376a7834b106f281b04f67b5d015c31732f594930f2ea81365f99d60c
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/

Dockerfile-release.arm64

@@ -1,4 +1,4 @@
-FROM --platform=linux/arm64 gcr.io/distroless/static-debian11@sha256:7e5c6a2a4ae854242874d36171b31d26e0539c98fc6080f942f16b03e82851ab
+FROM --platform=linux/arm64 gcr.io/distroless/static-debian11@sha256:6d31326376a7834b106f281b04f67b5d015c31732f594930f2ea81365f99d60c
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/

Dockerfile-release.ppc64le

@@ -1,4 +1,4 @@
-FROM --platform=linux/ppc64le gcr.io/distroless/static-debian11@sha256:7e5c6a2a4ae854242874d36171b31d26e0539c98fc6080f942f16b03e82851ab
+FROM --platform=linux/ppc64le gcr.io/distroless/static-debian11@sha256:6d31326376a7834b106f281b04f67b5d015c31732f594930f2ea81365f99d60c
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/

Dockerfile-release.s390x

@@ -1,4 +1,4 @@
-FROM --platform=linux/s390x gcr.io/distroless/static-debian11@sha256:7e5c6a2a4ae854242874d36171b31d26e0539c98fc6080f942f16b03e82851ab
+FROM --platform=linux/s390x gcr.io/distroless/static-debian11@sha256:6d31326376a7834b106f281b04f67b5d015c31732f594930f2ea81365f99d60c
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/

Makefile

@@ -571,4 +571,27 @@ pull-docker-functional:
 GOFAIL_VERSION = $(shell cd tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail)
 .PHONY: install-gofail
 install-gofail:
-	go install go.etcd.io/gofail@${GOFAIL_VERSION}
+	go install go.etcd.io/gofail@${GOFAIL_VERSION}
+
+.PHONY: gofail-enable
+gofail-enable: install-gofail
+	PASSES="toggle_failpoints" FAILPOINTS=true ./test.sh
+
+.PHONY: gofail-disable
+gofail-disable: install-gofail
+	PASSES="toggle_failpoints" ./test.sh
+
+.PHONY: verify
+verify: verify-go-versions
+
+.PHONY: verify-go-versions
+verify-go-versions:
+	./scripts/verify_go_versions.sh
+
+.PHONY: fix
+fix: sync-toolchain-directive
+	./scripts/fix.sh
+
+.PHONY: sync-toolchain-directive
+sync-toolchain-directive:
+	./scripts/sync_go_toolchain_directive.sh

api/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/api/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/coreos/go-semver v0.3.0
 	github.com/gogo/protobuf v1.3.2

api/version/version.go

@@ -26,7 +26,7 @@ import (
 var (
 	// MinClusterVersion is the min cluster version this etcd binary is compatible with.
 	MinClusterVersion = "3.0.0"
-	Version           = "3.5.13"
+	Version           = "3.5.14"
 	APIVersion        = "unknown"
 
 	// Git SHA Value will be set during build

build.sh

@@ -27,7 +27,7 @@ GOFAIL_VERSION=$(cd tools/mod && go list -m -f '{{.Version}}' go.etcd.io/gofail)
 toggle_failpoints() {
   mode="$1"
   if command -v gofail >/dev/null 2>&1; then
-    run gofail "$mode" server/etcdserver/ server/mvcc/ server/wal/ server/mvcc/backend/
+    run gofail "$mode" server/etcdserver/ server/lease/leasehttp server/mvcc/ server/wal/ server/mvcc/backend/
     if [[ "$mode" == "enable" ]]; then
       go get go.etcd.io/gofail@"${GOFAIL_VERSION}"
       cd ./server && go get go.etcd.io/gofail@"${GOFAIL_VERSION}"

client/pkg/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/pkg/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/stretchr/testify v1.8.4

client/v2/go.mod

@@ -2,11 +2,13 @@ module go.etcd.io/etcd/client/v2
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/json-iterator/go v1.1.11
 	github.com/modern-go/reflect2 v1.0.1
-	go.etcd.io/etcd/api/v3 v3.5.13
-	go.etcd.io/etcd/client/pkg/v3 v3.5.13
+	go.etcd.io/etcd/api/v3 v3.5.14
+	go.etcd.io/etcd/client/pkg/v3 v3.5.14
 )
 
 require (

client/v3/client.go

@@ -86,7 +86,7 @@ func New(cfg Config) (*Client, error) {
 // service interface implementations and do not need connection management.
 func NewCtxClient(ctx context.Context, opts ...Option) *Client {
 	cctx, cancel := context.WithCancel(ctx)
-	c := &Client{ctx: cctx, cancel: cancel, lgMu: new(sync.RWMutex)}
+	c := &Client{ctx: cctx, cancel: cancel, lgMu: new(sync.RWMutex), mu: new(sync.RWMutex)}
 	for _, opt := range opts {
 		opt(c)
 	}

client/v3/go.mod

@@ -2,12 +2,14 @@ module go.etcd.io/etcd/client/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/dustin/go-humanize v1.0.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/prometheus/client_golang v1.11.1
-	go.etcd.io/etcd/api/v3 v3.5.13
-	go.etcd.io/etcd/client/pkg/v3 v3.5.13
+	go.etcd.io/etcd/api/v3 v3.5.14
+	go.etcd.io/etcd/client/pkg/v3 v3.5.14
 	go.uber.org/zap v1.17.0
 	google.golang.org/grpc v1.59.0
 	sigs.k8s.io/yaml v1.2.0

client/v3/retry_interceptor.go

@@ -19,6 +19,7 @@ package clientv3
 
 import (
 	"context"
+	"errors"
 	"io"
 	"sync"
 	"time"
@@ -85,7 +86,7 @@ func (c *Client) unaryClientInterceptor(optFuncs ...retryOption) grpc.UnaryClien
 				}
 				continue
 			}
-			if !isSafeRetry(c.lg, lastErr, callOpts) {
+			if !isSafeRetry(c, lastErr, callOpts) {
 				return lastErr
 			}
 		}
@@ -279,7 +280,7 @@ func (s *serverStreamingRetryingStream) receiveMsgAndIndicateRetry(m interface{}
 		return true, err
 
 	}
-	return isSafeRetry(s.client.lg, err, s.callOpts), err
+	return isSafeRetry(s.client, err, s.callOpts), err
 }
 
 func (s *serverStreamingRetryingStream) reestablishStreamAndResendBuffer(callCtx context.Context) (grpc.ClientStream, error) {
@@ -319,17 +320,28 @@ func waitRetryBackoff(ctx context.Context, attempt uint, callOpts *options) erro
 }
 
 // isSafeRetry returns "true", if request is safe for retry with the given error.
-func isSafeRetry(lg *zap.Logger, err error, callOpts *options) bool {
+func isSafeRetry(c *Client, err error, callOpts *options) bool {
 	if isContextError(err) {
 		return false
 	}
+
+	// Situation when learner refuses RPC it is supposed to not serve is from the server
+	// perspective not retryable.
+	// But for backward-compatibility reasons we need  to support situation that
+	// customer provides mix of learners (not yet voters) and voters with an
+	// expectation to pick voter in the next attempt.
+	// TODO: Ideally client should be 'aware' which endpoint represents: leader/voter/learner with high probability.
+	if errors.Is(err, rpctypes.ErrGPRCNotSupportedForLearner) && len(c.Endpoints()) > 1 {
+		return true
+	}
+
 	switch callOpts.retryPolicy {
 	case repeatable:
 		return isSafeRetryImmutableRPC(err)
 	case nonRepeatable:
 		return isSafeRetryMutableRPC(err)
 	default:
-		lg.Warn("unrecognized retry policy", zap.String("retryPolicy", callOpts.retryPolicy.String()))
+		c.lg.Warn("unrecognized retry policy", zap.String("retryPolicy", callOpts.retryPolicy.String()))
 		return false
 	}
 }

etcd.conf.yml.sample

@@ -125,6 +125,12 @@ peer-transport-security:
   # Peer TLS using generated certificates.
   auto-tls: false
 
+  # Allowed CN for inter peer authentication.
+  allowed-cn:
+
+  # Allowed TLS hostname for inter peer authentication.
+  allowed-hostname:
+
 # The validity period of the self-signed certificate, the unit is year.
 self-signed-cert-validity: 1