From 85f8fdec5a4b595a74558822bb7b541ad387cbd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 28 Oct 2021 10:03:35 +0200 Subject: [PATCH] docs: simplify dev guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We removed the obsolete `minikube.sh` script for a cleaner dev procedure. Closes: https://github.com/rook/rook/issues/8851 Signed-off-by: Sébastien Han --- .../integration-test-helm-suite.yaml | 2 +- .../integration-tests-on-release.yaml | 2 +- tests/README.md | 60 +++++++----- tests/scripts/github-action-helper.sh | 6 ++ tests/scripts/minikube.sh | 96 ------------------- 5 files changed, 44 insertions(+), 122 deletions(-) delete mode 100755 tests/scripts/minikube.sh diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml index fc1d7e88d1b8..20196903dd2b 100644 --- a/.github/workflows/integration-test-helm-suite.yaml +++ b/.github/workflows/integration-test-helm-suite.yaml @@ -52,7 +52,7 @@ jobs: - name: TestCephHelmSuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - tests/scripts/minikube.sh helm + tests/scripts/github-action-helper.sh create_helm_tag tests/scripts/helm.sh up export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) SKIP_TEST_CLEANUP=false SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -failfast -run CephHelmSuite github.com/rook/rook/tests/integration diff --git a/.github/workflows/integration-tests-on-release.yaml b/.github/workflows/integration-tests-on-release.yaml index 530c4ef45269..a04747139ba6 100644 --- a/.github/workflows/integration-tests-on-release.yaml +++ b/.github/workflows/integration-tests-on-release.yaml @@ -53,7 +53,7 @@ jobs: - name: TestCephHelmSuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - tests/scripts/minikube.sh helm + tests/scripts/github-action-helper.sh create_helm_tag tests/scripts/helm.sh up export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) SKIP_TEST_CLEANUP=false SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -run CephHelmSuite github.com/rook/rook/tests/integration diff --git a/tests/README.md b/tests/README.md index b20eb957d002..50abe1af17be 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,33 +6,37 @@ quickly spin up a Kubernetes cluster. The Test framework is designed to install ## Install Kubernetes -You can choose any Kubernetes install of your choice. The test framework only depends on `kubectl` being configured. -An optional helper script is provided to install Kubernetes with minikube. +You can choose any Kubernetes install of your choice. The test framework only depends on `kubectl` +being configured. To install `kubectl`, please see the [official guide](https://kubernetes.io/docs/tasks/tools/#kubectl). ### Minikube +To install Minikube follow the [official +guide](https://minikube.sigs.k8s.io/docs/start/). It is recommended to use the +kvm2 driver when running on a Linux machine and the hyperkit driver when running on a MacOS. Both +allow to create and attach additional disks to the virtual machine. This is required for the Ceph +OSD to consume one drive. We don't recommend any other drivers for Rook. You will need a Minikube +version 1.23 or higher. + Starting the cluster on Minikube is as simple as running: ```console -tests/scripts/minikube.sh up -``` +# On Linux +minikube start --disk-size=40g --extra-disks=1 --driver kvm2 -To copy Rook images generated from your local build into the Minikube VM, run the following commands after `minikube.sh up` succeeded: - -```console -tests/scripts/minikube.sh update -tests/scripts/minikube.sh helm +# On MacOS +minikube start --disk-size=40g --extra-disks=1 --driver hyperkit ``` -Stopping the cluster and destroying the Minikube VM can be done with: +It is recommended to install a Docker client on your host system too. Depending on your operating +system follow the [official guide](https://docs.docker.com/engine/install/binaries/). + +Stopping the cluster and destroying the Minikube virtual machine can be done with: ```console -tests/scripts/minikube.sh clean +minikube delete ``` -For faster iteration during development, take a snapshot of the fresh minikube VM and then reset -the VM each time you need a clean environment. - ## Install Helm Use [helm.sh](/tests/scripts/helm.sh) to install Helm and set up Rook charts defined under `_output/charts` (generated by build): @@ -41,27 +45,35 @@ Use [helm.sh](/tests/scripts/helm.sh) to install Helm and set up Rook charts def - To clean up `tests/scripts/helm.sh clean`. **NOTE:** These helper scripts depend on some artifacts under the `_output/` directory generated during build time. -These scripts should be run from the project root. e.g., `tests/script/minikube.sh up`. +These scripts should be run from the project root. **NOTE**: If Helm is not available in your `PATH`, Helm will be downloaded to a temporary directory (`/tmp/rook-tests-scripts-helm`) and used from that directory. ## Run Tests -From the project root do the following after bringing up your Kubernetes cluster such as minikube: - ### 1. Build rook -`make build` +Now that the Kubernetes cluster is running we need to populate the Docker registry environment variables: -### 2. Start Kubernetes +```console +eval $(minikube docker-env -p minikube) +``` + +The build process will build and push the images to the Docker registry **inside** the Minikube +virtual machine. + +```console +make build +``` -Start the minikube cluster and copy the local build into minikube: +Now tag the newly built images to `rook/ceph:local-build` for running tests, or `rook/ceph:master` if creating sample manifests:: ```console -tests/scripts/minikube.sh up +docker tag $(docker images|awk '/build-/ {print $1}') rook/ceph:local-build +docker tag rook/ceph:local-build rook/ceph:master ``` -### 3. Run integration tests +### 2. Run integration tests Some settings are available to run the tests under different environments. The settings are all configured with environment variables. See [environment.go](/tests/framework/installer/environment.go) for the available environment variables. @@ -70,7 +82,7 @@ At least you should set the following variables. ```console export TEST_HELM_PATH=/tmp/rook-tests-scripts-helm/linux-amd64/helm export TEST_BASE_DIR=WORKING_DIR -export TEST_SCRATCH_DEVICE= # for example, TEST_SCRATCH_DEVICE=/dev/sdb +export TEST_SCRATCH_DEVICE=/dev/vdb ``` Please note that the integration tests erase the contents of TEST_SCRATCH_DEVICE. @@ -95,7 +107,7 @@ To run specific tests inside a suite: go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration -testify.m TestARookClusterInstallation_SmokeTest ``` -### To run tests on OpenShift environment +### 3. To run tests on OpenShift environment - Setup OpenShift environment and export KUBECONFIG before executing the tests. - Make sure `oc` executable file is in the PATH. diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 8574b2c28c73..1f6890a357b4 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -350,6 +350,12 @@ EOF diff 1M.dat 1M-get.dat } +function create_helm_tag() { + helm_tag="$(cat _output/version)" + build_image="$(docker images | awk '/build-/ {print $1}')" + docker tag "${build_image}" "rook/ceph:${helm_tag}" +} + FUNCTION="$1" shift # remove function arg now that we've recorded it # call the function with the remainder of the user-provided args diff --git a/tests/scripts/minikube.sh b/tests/scripts/minikube.sh deleted file mode 100755 index 5e082de28f0d..000000000000 --- a/tests/scripts/minikube.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash -e - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# source=build/common.sh -source "${scriptdir}/../../build/common.sh" - -function wait_for_ssh() { - local tries=100 - while (( tries > 0 )) ; do - if minikube ssh echo connected &> /dev/null ; then - return 0 - fi - tries=$(( tries - 1 )) - sleep 0.1 - done - echo ERROR: ssh did not come up >&2 - exit 1 -} - -function copy_image_to_cluster() { - local build_image=$1 - local final_image=$2 - ${DOCKERCMD} tag "${build_image}" "${final_image}" - minikube image load "${final_image}" -} - -function copy_images() { - if [[ "$1" == "" || "$1" == "ceph" ]]; then - echo "copying ceph images" - copy_image_to_cluster "${BUILD_REGISTRY}/ceph-amd64" rook/ceph:master - fi - - if [[ "$1" == "" || "$1" == "cassandra" ]]; then - echo "copying cassandra image" - copy_image_to_cluster "${BUILD_REGISTRY}/cassandra-amd64" rook/cassandra:master - fi - - if [[ "$1" == "" || "$1" == "nfs" ]]; then - echo "copying nfs image" - copy_image_to_cluster "${BUILD_REGISTRY}/nfs-amd64" rook/nfs:master - fi -} - -# configure minikube -MEMORY=${MEMORY:-"3000"} - -# use vda1 instead of sda1 when running with the libvirt driver -VM_DRIVER=$(minikube config get driver 2>/dev/null || echo "virtualbox") -if [[ "$VM_DRIVER" == "kvm2" ]]; then - DISK="vda1" -else - DISK="sda1" -fi - -case "${1:-}" in - up) - echo "starting minikube with kubeadm bootstrapper" - minikube start --memory="${MEMORY}" -b kubeadm --vm-driver="${VM_DRIVER}" - wait_for_ssh - # create a link so the default dataDirHostPath will work for this environment - minikube ssh "sudo mkdir -p /mnt/${DISK}/rook/ && sudo ln -sf /mnt/${DISK}/rook/ /var/lib/" - copy_images "$2" - ;; - down) - minikube stop - ;; - ssh) - echo "connecting to minikube" - minikube ssh - ;; - update) - copy_images "$2" - ;; - wordpress) - echo "copying the wordpress images" - copy_image_to_cluster mysql:5.6 mysql:5.6 - copy_image_to_cluster wordpress:4.6.1-apache wordpress:4.6.1-apache - ;; - helm) - echo " copying rook image for helm" - helm_tag="$(cat _output/version)" - copy_image_to_cluster "${BUILD_REGISTRY}/ceph-amd64" "rook/ceph:${helm_tag}" - ;; - clean) - minikube delete - ;; - *) - echo "usage:" >&2 - echo " $0 up [ceph | cassandra | nfs]" >&2 - echo " $0 down" >&2 - echo " $0 clean" >&2 - echo " $0 ssh" >&2 - echo " $0 update [ceph | cassandra | nfs]" >&2 - echo " $0 wordpress" >&2 - echo " $0 helm" >&2 -esac