Use cloudbuild, add Makefile targets, and build script for envtest

Signed-off-by: Vince Prignano <vincepri@redhat.com>

.gitignore

@@ -18,6 +18,7 @@
 *~
 
 # Tools binaries.
+out
 hack/tools/bin
 
 junit-report.xml

Makefile

@@ -92,3 +92,50 @@ clean: ## Cleanup.
 .PHONY: clean-bin
 clean-bin: ## Remove all generated binaries.
 	rm -rf hack/tools/bin
+
+.PHONE: clean-release
+clean-release: ## Remove all generated release binaries.
+	rm -rf $(RELEASE_DIR)
+
+## --------------------------------------
+## Envtest Build
+## --------------------------------------
+
+RELEASE_DIR := out
+
+.PHONY: $(RELEASE_DIR)
+$(RELEASE_DIR):
+	mkdir -p $(RELEASE_DIR)/
+
+.PHONY: release-envtest
+release-envtest:
+	./hack/envtest/build.sh
+
+.PHONY: release-package-envtest
+release-package-envtest: clean-release ## Build the envtest binaries by operating system.
+	OS=linux ARCH=amd64 $(MAKE) release-envtest-docker-build
+	OS=linux ARCH=arm64 $(MAKE) release-envtest-docker-build
+	OS=linux ARCH=ppc64le $(MAKE) release-envtest-docker-build
+	OS=linux ARCH=s390x $(MAKE) release-envtest-docker-build
+	OS=darwin ARCH=amd64 $(MAKE) release-envtest-docker-build
+	OS=darwin ARCH=arm64 $(MAKE) release-envtest-docker-build
+	OS=windows ARCH=amd64 $(MAKE) release-envtest-docker-build
+
+.PHONY: release-envtest-docker-build
+release-envtest-docker-build: $(RELEASE_DIR) ## Build the envtest binaries.
+	@: $(if $(GO_VERSION),,$(error GO_VERSION is not set))
+	@: $(if $(KUBERNETES_VERSION),,$(error KUBERNETES_VERSION is not set))
+	@: $(if $(ETCD_VERSION),,$(error ETCD_VERSION is not set))
+	@: $(if $(OS),,$(error OS is not set))
+	@: $(if $(ARCH),,$(error ARCH is not set))
+
+	docker buildx build \
+		--file ./hack/envtest/$(OS)/Dockerfile \
+		--build-arg GO_VERSION=$(GO_VERSION) \
+		--build-arg KUBERNETES_VERSION=$(KUBERNETES_VERSION) \
+		--build-arg ETCD_VERSION=$(ETCD_VERSION) \
+		--build-arg OS=$(OS) \
+		--build-arg ARCH=$(ARCH) \
+		--tag sigs.k8s.io/controller-tools/envtest:$(KUBERNETES_VERSION)-$(OS)-$(ARCH) \
+		--output type=local,dest=$(RELEASE_DIR) \
+		.

cloudbuild.yaml

@@ -0,0 +1,20 @@
+# See https://cloud.google.com/cloud-build/docs/build-config
+# See https://console.cloud.google.com/gcr/images/k8s-staging-test-infra/global/gcb-docker-gcloud
+timeout: 2700s
+options:
+  substitution_option: ALLOW_LOOSE
+  machineType: 'E2_HIGHCPU_8'
+steps:
+  - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20230522-312425ae46'
+    entrypoint: make
+    env:
+    - DOCKER_CLI_EXPERIMENTAL=enabled
+    - TAG=$_GIT_TAG
+    - PULL_BASE_REF=$_PULL_BASE_REF
+    - DOCKER_BUILDKIT=1
+    args: ['release-envtest', '-j', '8']
+substitutions:
+  # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
+  # can be used as a substitution
+  _GIT_TAG: '12345'
+  _PULL_BASE_REF: 'dev'

hack/envtest/_matrix/v1.28.0.yaml

@@ -1,2 +1,3 @@
 go: 1.21
 etcd: v3.5.9
+

hack/envtest/build.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -x
+
+# This script is used to build the test binaries for the envtest package.
+# It's intended to be run from the root of the repository, through the Makefile.
+
+envtest_dir=$(dirname ${BASH_SOURCE})
+ROOT=$(dirname "${BASH_SOURCE[0]}")/../../..
+
+# Check modified files in hack/envtest/_matrix/*.yaml
+# If there are any changes, we need to rebuild the test binaries.
+# This is to ensure that the test binaries are always up-to-date with the matrix files.
+
+changed_files=$(git diff --name-only --relative hack/envtest/_matrix | grep -E '\.yaml$' || true)
+
+# If there are no changes, we don't need to rebuild the test binaries.
+if [ -z "${changed_files}" ]; then
+  exit 0
+fi
+
+# For each changed file, grab the desired Kubernetes patch version from the name of the file.
+for file in ${changed_files}; do
+  KUBERNETES_VERSION=$(basename "${file}" .yaml)
+  echo "Building test binaries for ${KUBERNETES_VERSION}"
+
+  GO_VERSION=$(yq eval .go "${file}")
+  ETCD_VERSION=$(yq eval .etcd "${file}")
+
+  make release-package-envtest \
+    KUBERNETES_VERSION="${KUBERNETES_VERSION}" \
+    GO_VERSION="${GO_VERSION}" \
+    ETCD_VERSION="${ETCD_VERSION}"
+done
+
+

hack/envtest/darwin/Dockerfile

@@ -51,6 +51,12 @@ RUN curl -sfLO https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/
     unzip -o ${ETCD_BASE_NAME}.zip && \
     cp ${ETCD_BASE_NAME}/etcd $DEST
 
+# Package into tarball.
+RUN tar -czvf /envtest-${KUBERNETES_VERSION}-${OS}-${ARCH}.tar.gz $DEST
+
 # Build the final image with the binaries.
 FROM scratch
-COPY --from=builder /controller-tools/envtest /controller-tools/envtest
+ARG OS
+ARG ARCH
+ARG KUBERNETES_VERSION
+COPY --from=builder /envtest-${KUBERNETES_VERSION}-${OS}-${ARCH}.tar.gz /

hack/envtest/linux/Dockerfile

@@ -45,6 +45,12 @@ RUN curl -sfLO https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/
     tar xzf ${ETCD_BASE_NAME}.tar.gz && \
     cp ${ETCD_BASE_NAME}/etcd $DEST
 
+# Package into tarball.
+RUN tar -czvf /envtest-${KUBERNETES_VERSION}-${OS}-${ARCH}.tar.gz $DEST
+
 # Build the final image with the binaries.
 FROM scratch
-COPY --from=builder /controller-tools/envtest /controller-tools/envtest
+ARG OS
+ARG ARCH
+ARG KUBERNETES_VERSION
+COPY --from=builder /envtest-${KUBERNETES_VERSION}-${OS}-${ARCH}.tar.gz /

hack/envtest/windows/Dockerfile

@@ -51,6 +51,12 @@ RUN curl -sfLO https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/
     unzip -o ${ETCD_BASE_NAME}.zip && \
     cp ${ETCD_BASE_NAME}/etcd.exe $DEST
 
+# Package into tarball.
+RUN tar -czvf /envtest-${KUBERNETES_VERSION}-${OS}-${ARCH}.tar.gz $DEST
+
 # Build the final image with the binaries.
 FROM scratch
-COPY --from=builder /controller-tools/envtest /controller-tools/envtest
+ARG OS
+ARG ARCH
+ARG KUBERNETES_VERSION
+COPY --from=builder /envtest-${KUBERNETES_VERSION}-${OS}-${ARCH}.tar.gz /