From e094139b05244c5eab0e62172f92ac5e90feb73b Mon Sep 17 00:00:00 2001 From: Madhav Jivrajani Date: Thu, 25 Apr 2024 14:12:51 +0530 Subject: [PATCH 1/2] scripts: default to using .go-version's version for tests and builds Additionally, provide ability to opt-out of the .go-version and use a custom one via env vars: FORCE_HOST_GO and GO_VERSION. Signed-off-by: Madhav Jivrajani --- scripts/test_lib.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/test_lib.sh b/scripts/test_lib.sh index b521a06ee58..41e9a5e1531 100644 --- a/scripts/test_lib.sh +++ b/scripts/test_lib.sh @@ -456,3 +456,27 @@ function git_assert_branch_in_sync { log_warning "Cannot verify consistency with the origin, as git is on detached branch." fi } + +# The version present in the .go-verion is the default version that test and build scripts will use. +# However, it is possible to control the version that should be used with the help of env vars: +# - FORCE_HOST_GO: if set to a non-empty value, use the version of go installed in system's $PATH. +# - GO_VERSION: desired version of go to be used, might differ from what is present in .go-version. +# If empty, the value defaults to the version in .go-version. +function determine_go_version { + # Borrowing from how Kubernetes does this: + # https://github.com/kubernetes/kubernetes/blob/17854f0e0a153b06f9d0db096e2cd8ab2fa89c11/hack/lib/golang.sh#L510-L520 + # + # default GO_VERSION to content of .go-version + GO_VERSION="${GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" + if [ "${GOTOOLCHAIN:-auto}" != 'auto' ]; then + # no-op, just respect GOTOOLCHAIN + : + elif [ -n "${FORCE_HOST_GO:-}" ]; then + export GOTOOLCHAIN='local' + else + GOTOOLCHAIN="go${GO_VERSION}" + export GOTOOLCHAIN + fi +} + +determine_go_version From f6a35f8786f1ecc2a1065ec78336567a33c6eed8 Mon Sep 17 00:00:00 2001 From: Madhav Jivrajani Date: Thu, 25 Apr 2024 14:13:24 +0530 Subject: [PATCH 2/2] .*: sync go toolchain version and add ability to verify versions This commit adds a script to sync the version present in .go-version across all go.mod files as the toolchain directive. As part of that, this commit also modifies go.mod files that did not have synced toolchain directives. Additionally, this also adds a script to verify all toolchain and go directives against the version present in .go-version as follows: (1) The go directive <= version in .go-version (2) The toolchain directive == version in .go-version This script runs as part of the `make verify` target, making it run as a presbumit by default. Signed-off-by: Madhav Jivrajani --- Makefile | 10 ++++- api/go.mod | 2 + client/internal/v2/go.mod | 2 + client/pkg/go.mod | 2 + client/v3/go.mod | 2 + etcdctl/go.mod | 2 + etcdutl/go.mod | 2 + pkg/go.mod | 2 + scripts/fix.sh | 2 + scripts/sync_go_toolchain_directive.sh | 14 +++++++ scripts/verify_go_versions.sh | 53 ++++++++++++++++++++++++++ server/go.mod | 2 + tests/go.mod | 2 + tools/rw-heatmaps/go.mod | 2 + tools/testgrid-analysis/go.mod | 2 + 15 files changed, 100 insertions(+), 1 deletion(-) create mode 100755 scripts/sync_go_toolchain_directive.sh create mode 100755 scripts/verify_go_versions.sh diff --git a/Makefile b/Makefile index 07bebc0dc43..50b9a31ff2e 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ fuzz: verify: verify-gofmt verify-bom verify-lint verify-dep verify-shellcheck verify-goword \ verify-govet verify-license-header verify-receiver-name verify-mod-tidy \ verify-shellws verify-proto-annotations verify-genproto verify-yamllint \ - verify-govet-shadow verify-markdown-marker + verify-govet-shadow verify-markdown-marker verify-go-versions .PHONY: fix fix: fix-bom fix-lint fix-yamllint @@ -187,3 +187,11 @@ clean: rm -rf ./tests/e2e/default.proxy rm -rf ./bin/shellcheck* find ./ -name "127.0.0.1:*" -o -name "localhost:*" -o -name "*.log" -o -name "agent-*" -o -name "*.coverprofile" -o -name "testname-proxy-*" -delete + +.PHONY: verify-go-versions +verify-go-versions: + ./scripts/verify_go_versions.sh + +.PHONY: sync-toolchain-directive +sync-toolchain-directive: + ./scripts/sync_go_toolchain_directive.sh diff --git a/api/go.mod b/api/go.mod index 708fd25f228..becaf89378b 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/api/v3 go 1.22 +toolchain go1.22.2 + require ( github.com/coreos/go-semver v0.3.1 github.com/gogo/protobuf v1.3.2 diff --git a/client/internal/v2/go.mod b/client/internal/v2/go.mod index b66163f6b32..b215067570a 100644 --- a/client/internal/v2/go.mod +++ b/client/internal/v2/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/v2 go 1.22 +toolchain go1.22.2 + require ( go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 diff --git a/client/pkg/go.mod b/client/pkg/go.mod index 397898bd155..1072a4231eb 100644 --- a/client/pkg/go.mod +++ b/client/pkg/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/pkg/v3 go 1.22 +toolchain go1.22.2 + require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/stretchr/testify v1.9.0 diff --git a/client/v3/go.mod b/client/v3/go.mod index 47fc7a682d8..8ef2e219feb 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/v3 go 1.22 +toolchain go1.22.2 + require ( github.com/coreos/go-semver v0.3.1 github.com/dustin/go-humanize v1.0.1 diff --git a/etcdctl/go.mod b/etcdctl/go.mod index b20ab98b3c4..6c816b4024c 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/etcdctl/v3 go 1.22 +toolchain go1.22.2 + require ( github.com/bgentry/speakeasy v0.1.0 github.com/cheggaaa/pb/v3 v3.1.5 diff --git a/etcdutl/go.mod b/etcdutl/go.mod index d0e66a078a0..a77dd870494 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/etcdutl/v3 go 1.22 +toolchain go1.22.2 + replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/client/pkg/v3 => ../client/pkg diff --git a/pkg/go.mod b/pkg/go.mod index 06af488e904..1b62117ea27 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/pkg/v3 go 1.22 +toolchain go1.22.2 + require ( github.com/creack/pty v1.1.18 github.com/dustin/go-humanize v1.0.1 diff --git a/scripts/fix.sh b/scripts/fix.sh index 7936a2cf3d4..165e201bfb4 100755 --- a/scripts/fix.sh +++ b/scripts/fix.sh @@ -36,5 +36,7 @@ run_for_modules run ${GO_CMD} fmt || exit 2 run_for_module tests bom_fix || exit 2 bash_ws_fix || exit 2 +log_callout "Syncing go toolchain directives" +run ./scripts/sync_go_toolchain_directive.sh || exit 2 log_success -e "\\nSUCCESS: etcd code is fixed :)" diff --git a/scripts/sync_go_toolchain_directive.sh b/scripts/sync_go_toolchain_directive.sh new file mode 100755 index 00000000000..643138e1d0a --- /dev/null +++ b/scripts/sync_go_toolchain_directive.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# This script looks at the version present in the .go-version file and treats +# that to be the value of the toolchain directive that go should use. It then +# updates the toolchain directives of all go.mod files to reflect this version. +# +# We do this to ensure that .go-version acts as the source of truth for go versions. + +set -euo pipefail + +source ./scripts/test_lib.sh + +TARGET_GO_VERSION="${TARGET_GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" +find . -name 'go.mod' -exec go mod edit -toolchain=go"${TARGET_GO_VERSION}" {} \; diff --git a/scripts/verify_go_versions.sh b/scripts/verify_go_versions.sh new file mode 100755 index 00000000000..11d34f92a51 --- /dev/null +++ b/scripts/verify_go_versions.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# This script verifies that the value of the toolchain directive in the +# go.mod files always match that of the .go-version file to ensure that +# we accidentally don't test and release with differing versions of Go. + +set -euo pipefail + +source ./scripts/test_lib.sh + +target_go_version="${target_go_version:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" +log_info "expected go toolchain directive: go${target_go_version}" +log_info + +toolchain_out_of_sync="false" +go_line_violation="false" + +# verify_go_versions takes a go.mod filepath as an argument +# and checks if: +# (1) go directive <= version in .go-version +# (2) toolchain directive == version in .go-version +function verify_go_versions() { + # shellcheck disable=SC2086 + toolchain_version="$(go mod edit -json $1 | jq -r .Toolchain)" + # shellcheck disable=SC2086 + go_line_version="$(go mod edit -json $1 | jq -r .Go)" + if [[ "go${target_go_version}" != "${toolchain_version}" ]]; then + log_error "go toolchain directive out of sync for $1, got: ${toolchain_version}" + toolchain_out_of_sync="true" + fi + if ! printf '%s\n' "${go_line_version}" "${target_go_version}" | sort --check=silent --version-sort; then + log_error "go directive in $1 is greater than maximum allowed: go${target_go_version}" + go_line_violation="true" + fi +} + +while read -r mod; do + verify_go_versions "${mod}"; +done < <(find . -name 'go.mod') + +if [[ "${toolchain_out_of_sync}" == "true" ]]; then + log_error + log_error "Please run scripts/sync_go_toolchain_directive.sh or update .go-version to rectify this error" +fi + +if [[ "${go_line_violation}" == "true" ]]; then + log_error + log_error "Please update .go-version to rectify this error, any go directive should be <= .go-version" +fi + +if [[ "${go_line_violation}" == "true" ]] || [[ "${toolchain_out_of_sync}" == "true" ]]; then + exit 1 +fi diff --git a/server/go.mod b/server/go.mod index 598364b11ce..a339af7bf19 100644 --- a/server/go.mod +++ b/server/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/server/v3 go 1.22 +toolchain go1.22.2 + require ( github.com/coreos/go-semver v0.3.1 github.com/coreos/go-systemd/v22 v22.5.0 diff --git a/tests/go.mod b/tests/go.mod index 7c59aa224ad..d5987ab1a4c 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/tests/v3 go 1.22 +toolchain go1.22.2 + replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/client/pkg/v3 => ../client/pkg diff --git a/tools/rw-heatmaps/go.mod b/tools/rw-heatmaps/go.mod index db64003f833..344f9a54382 100644 --- a/tools/rw-heatmaps/go.mod +++ b/tools/rw-heatmaps/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/tools/rw-heatmaps/v3 go 1.22 +toolchain go1.22.2 + require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/tools/testgrid-analysis/go.mod b/tools/testgrid-analysis/go.mod index 0e0056067f2..75b8e3884c2 100644 --- a/tools/testgrid-analysis/go.mod +++ b/tools/testgrid-analysis/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/tools/testgrid-analysis/v3 go 1.22 +toolchain go1.22.2 + require ( github.com/GoogleCloudPlatform/testgrid v0.0.173 github.com/google/go-github/v60 v60.0.0