From 94c9b13fbd9dc8d5d42207187e1855d11779bc61 Mon Sep 17 00:00:00 2001 From: Sunny Date: Mon, 8 Jan 2024 14:35:20 +0000 Subject: [PATCH] tests/int: Add separate resource cleanup step Introduce a destroy-only mode in the test runner to run terraform destroy for the respective cloud provider configurations. This can be used to destroy cloud resources without going through the whole provision-test process. Add a new step in github actions workflow to run the test binary in destoy-only mode at the very end irrespective of the result of the previous steps. This ensures that the infrastructure is always destroyed, even if the CI job is cancelled. Signed-off-by: Sunny --- .github/workflows/e2e-azure.yaml | 11 +++++++++++ .github/workflows/e2e-gcp.yaml | 10 ++++++++++ tests/integration/Makefile | 8 ++++++++ tests/integration/README.md | 15 ++++++++++++--- tests/integration/go.mod | 4 ++-- tests/integration/go.sum | 4 ++-- tests/integration/suite_test.go | 15 +++++++++++++++ 7 files changed, 60 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-azure.yaml b/.github/workflows/e2e-azure.yaml index 67743f3f51..87d7dfe184 100644 --- a/.github/workflows/e2e-azure.yaml +++ b/.github/workflows/e2e-azure.yaml @@ -123,3 +123,14 @@ jobs: echo $GITREPO_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub export GITREPO_SSH_PUB_PATH=build/ssh/key.pub make test-azure + - name: Ensure resource cleanup + if: ${{ always() }} + env: + ARM_CLIENT_ID: ${{ secrets.AZ_ARM_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZ_ARM_CLIENT_SECRET }} + ARM_SUBSCRIPTION_ID: ${{ secrets.AZ_ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.AZ_ARM_TENANT_ID }} + TF_VAR_azuredevops_org: ${{ secrets.TF_VAR_azuredevops_org }} + TF_VAR_azuredevops_pat: ${{ secrets.TF_VAR_azuredevops_pat }} + TF_VAR_location: ${{ vars.TF_VAR_azure_location }} + run: source .env && make destroy-azure diff --git a/.github/workflows/e2e-gcp.yaml b/.github/workflows/e2e-gcp.yaml index 1e85a4cc90..068968f2ec 100644 --- a/.github/workflows/e2e-gcp.yaml +++ b/.github/workflows/e2e-gcp.yaml @@ -90,3 +90,13 @@ jobs: echo $GITREPO_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub export GITREPO_SSH_PUB_PATH=build/ssh/key.pub make test-gcp + - name: Ensure resource cleanup + if: ${{ always() }} + env: + TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }} + TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }} + TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }} + TF_VAR_gcp_email: ${{ secrets.TF_VAR_gcp_email }} + TF_VAR_gcp_keyring: ${{ secrets.TF_VAR_gcp_keyring }} + TF_VAR_gcp_crypto_key: ${{ secrets.TF_VAR_gcp_crypto_key }} + run: source .env && make destroy-gcp diff --git a/tests/integration/Makefile b/tests/integration/Makefile index a924232fb6..ddae646ae5 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -17,6 +17,14 @@ test-azure: test-gcp: $(MAKE) test PROVIDER_ARG="-provider gcp" +destroy: + go test -timeout $(TEST_TIMEOUT) -v ./ $(GO_TEST_ARGS) $(PROVIDER_ARG) -destroy-only + +destroy-azure: + $(MAKE) destroy PROVIDER_ARG="-provider azure" + +destroy-gcp: + $(MAKE) destroy PROVIDER_ARG="-provider gcp" sops-check: ifeq ($(shell which sops),) diff --git a/tests/integration/README.md b/tests/integration/README.md index db6f8d1480..0fa1f3e32c 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -74,6 +74,10 @@ variables using use the terraform configuration below. Please make sure all the requirements of azure-gh-actions are followed before running it. +**NOTE:** When running the following for a repo under an organization, set the +environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the +`github` provider doesn't work. + ```hcl provider "github" { owner = "fluxcd" @@ -218,7 +222,7 @@ variables using use the terraform configuration below. Please make sure all the requirements of gcp-gh-actions are followed before running it. -**NOTE:** When running the following for a repo under and organization, set the +**NOTE:** When running the following for a repo under an organization, set the environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the `github` provider doesn't work. @@ -345,8 +349,13 @@ We also pull two version of `ghcr.io/stefanprodan/podinfo` image. These images a Container Registry and used to test `ImageRepository` and `ImageUpdateAutomation`. The terraform resources get created and the tests are run. -**IMPORTANT:** In case the terraform infrastructure results in a bad state, maybe due to a crash during the apply, -the whole infrastructure can be destroyed by running terraform destroy in terraform/ directory. +If not configured explicitly to retain the infrastructure, at the end of the +test, the test infrastructure is deleted. In case of any failure due to which +the resources don't get deleted, the `make destroy-*` commands can be run for +the respective provider. This will run terraform destroy in the respective +provider's terraform configuration directory. This can be used to quickly +destroy the infrastructure without going through the provision-test-destroy +steps. ### Debugging the tests diff --git a/tests/integration/go.mod b/tests/integration/go.mod index 4b361f81e4..3f9a6d708a 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -17,9 +17,10 @@ require ( github.com/fluxcd/pkg/git/gogit v0.16.1 github.com/fluxcd/pkg/runtime v0.43.2 github.com/fluxcd/source-controller/api v1.2.3 - github.com/fluxcd/test-infra/tftestenv v0.0.0-20230831142147-627bca8e7916 + github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa github.com/go-git/go-git/v5 v5.11.0 github.com/google/go-containerregistry v0.16.1 + github.com/hashicorp/terraform-exec v0.18.1 github.com/hashicorp/terraform-json v0.16.0 github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 github.com/onsi/gomega v1.30.0 @@ -89,7 +90,6 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hc-install v0.5.0 // indirect - github.com/hashicorp/terraform-exec v0.18.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/tests/integration/go.sum b/tests/integration/go.sum index c5c1509a5a..06fd3947de 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -153,8 +153,8 @@ github.com/fluxcd/pkg/version v0.2.2 h1:ZpVXECeLA5hIQMft11iLp6gN3cKcz6UNuVTQPw/b github.com/fluxcd/pkg/version v0.2.2/go.mod h1:NGnh/no8S6PyfCDxRFrPY3T5BUnqP48MxfxNRU0z8C0= github.com/fluxcd/source-controller/api v1.2.3 h1:71mXv3Qg9HEhcpqOq1ObmoE+P/HuZNaAvxfI7dqZMo8= github.com/fluxcd/source-controller/api v1.2.3/go.mod h1:5gaIVVH7hgb8p3HKFp8P6hGmZEC8fKSt4EcrG3g5vZI= -github.com/fluxcd/test-infra/tftestenv v0.0.0-20230831142147-627bca8e7916 h1:w9UGknpfAGbiObQALZiuWYGeriAU1wKCFTmI2tj/96M= -github.com/fluxcd/test-infra/tftestenv v0.0.0-20230831142147-627bca8e7916/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug= +github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa h1:JdI+rVwGF5gBYt+UBijOVzXtq7aAU80vgksMNXSCCfU= +github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= diff --git a/tests/integration/suite_test.go b/tests/integration/suite_test.go index 0e2b21860f..c20526998a 100644 --- a/tests/integration/suite_test.go +++ b/tests/integration/suite_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes/scheme" @@ -197,6 +198,20 @@ func TestMain(m *testing.M) { log.Fatalf("Failed to get provider config for %q", infraOpts.Provider) } + // Run destroy-only mode if enabled. + if infraOpts.DestroyOnly { + log.Println("Running in destroy-only mode...") + envOpts := []tftestenv.EnvironmentOption{ + tftestenv.WithVerbose(infraOpts.Verbose), + // Ignore any state lock in destroy-only mode. + tftestenv.WithTfDestroyOptions(tfexec.Lock(false)), + } + if err := tftestenv.Destroy(ctx, providerCfg.terraformPath, envOpts...); err != nil { + panic(err) + } + os.Exit(0) + } + // Initialize with non-zero exit code to indicate failure by default unless // set by a successful test run. exitCode := 1