From fd10d98dc60d149d41377e97d1f14e4c657a4823 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 26 Oct 2021 13:23:11 -0600 Subject: [PATCH] core: treat cluster as not existing if the cleanup policy is set The cluster CR can be forcefully deleted and cleanup the cluster resources if the yes-really-destroy-data policy is set on the CR. In this case, the other controllers should treat the cluster CR as not existing and allow the finalizers to be removed on those resources if they are requested for deletion. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/client/controller.go | 2 +- pkg/operator/ceph/cluster/rbd/controller.go | 2 +- .../ceph/controller/controller_utils.go | 11 ++- .../ceph/controller/controller_utils_test.go | 91 +++++++++++++++++++ pkg/operator/ceph/file/controller.go | 2 +- pkg/operator/ceph/file/mirror/controller.go | 2 +- pkg/operator/ceph/nfs/controller.go | 2 +- pkg/operator/ceph/object/controller.go | 2 +- pkg/operator/ceph/object/realm/controller.go | 2 +- pkg/operator/ceph/object/user/controller.go | 2 +- pkg/operator/ceph/object/zone/controller.go | 2 +- .../ceph/object/zonegroup/controller.go | 2 +- pkg/operator/ceph/pool/controller.go | 2 +- 13 files changed, 110 insertions(+), 14 deletions(-) diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go index bd36e33c86d9..1b2af01a472e 100644 --- a/pkg/operator/ceph/client/controller.go +++ b/pkg/operator/ceph/client/controller.go @@ -152,7 +152,7 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re } // Make sure a CephCluster is present otherwise do nothing - _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deletePool() function since everything is gone already diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index 8411cbf51508..2041bb0975f2 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -172,7 +172,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { logger.Debugf("CephCluster resource not ready in namespace %q, retrying in %q.", request.NamespacedName.Namespace, reconcileResponse.RequeueAfter.String()) return reconcileResponse, nil diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index ed399c761a78..2bb5c6e1684c 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -25,7 +25,6 @@ import ( "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -114,7 +113,7 @@ func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, control } // IsReadyToReconcile determines if a controller is ready to reconcile or not -func IsReadyToReconcile(ctx context.Context, c client.Client, clustercontext *clusterd.Context, namespacedName types.NamespacedName, controllerName string) (cephv1.CephCluster, bool, bool, reconcile.Result) { +func IsReadyToReconcile(ctx context.Context, c client.Client, namespacedName types.NamespacedName, controllerName string) (cephv1.CephCluster, bool, bool, reconcile.Result) { cephClusterExists := false // Running ceph commands won't work and the controller will keep re-queuing so I believe it's fine not to check @@ -130,9 +129,15 @@ func IsReadyToReconcile(ctx context.Context, c client.Client, clustercontext *cl logger.Debugf("%q: no CephCluster resource found in namespace %q", controllerName, namespacedName.Namespace) return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady } - cephClusterExists = true cephCluster = clusterList.Items[0] + // If the cluster has a cleanup policy to destroy the cluster and it has been marked for deletion, treat it as if it does not exist + if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.DeletionTimestamp.IsZero() { + logger.Infof("%q: CephCluster %q has a destructive cleanup policy, allowing resources to be deleted", controllerName, namespacedName) + return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady + } + + cephClusterExists = true logger.Debugf("%q: CephCluster resource %q found in namespace %q", controllerName, cephCluster.Name, namespacedName.Namespace) // read the CR status of the cluster diff --git a/pkg/operator/ceph/controller/controller_utils_test.go b/pkg/operator/ceph/controller/controller_utils_test.go index 52dac34a35bc..d8872551084c 100644 --- a/pkg/operator/ceph/controller/controller_utils_test.go +++ b/pkg/operator/ceph/controller/controller_utils_test.go @@ -17,12 +17,18 @@ limitations under the License. package controller import ( + ctx "context" "testing" "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/util/exec" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func CreateTestClusterFromStatusDetails(details map[string]cephv1.CephHealthMessage) cephv1.CephCluster { @@ -85,3 +91,88 @@ func TestSetCephCommandsTimeout(t *testing.T) { SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "1"}) assert.Equal(t, 1*time.Second, exec.CephCommandsTimeout) } + +func TestIsReadyToReconcile(t *testing.T) { + scheme := scheme.Scheme + scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) + + controllerName := "testing" + clusterName := types.NamespacedName{Name: "mycluster", Namespace: "myns"} + + t.Run("non-existent cephcluster", func(t *testing.T) { + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build() + c, ready, clusterExists, reconcileResult := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.False(t, clusterExists) + assert.Equal(t, WaitForRequeueIfCephClusterNotReady, reconcileResult) + }) + + t.Run("valid cephcluster", func(t *testing.T) { + cephCluster := &cephv1.CephCluster{} + objects := []runtime.Object{cephCluster} + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() + c, ready, clusterExists, reconcileResult := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.False(t, clusterExists) + assert.Equal(t, WaitForRequeueIfCephClusterNotReady, reconcileResult) + }) + + t.Run("deleted cephcluster with no cleanup policy", func(t *testing.T) { + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName.Name, + Namespace: clusterName.Namespace, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + }, + } + + objects := []runtime.Object{cephCluster} + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() + c, ready, clusterExists, reconcileResult := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.True(t, clusterExists) + assert.Equal(t, WaitForRequeueIfCephClusterNotReady, reconcileResult) + }) + + t.Run("cephcluster with cleanup policy when not deleted", func(t *testing.T) { + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName.Name, + Namespace: clusterName.Namespace, + }, + Spec: cephv1.ClusterSpec{ + CleanupPolicy: cephv1.CleanupPolicySpec{ + Confirmation: cephv1.DeleteDataDirOnHostsConfirmation, + }, + }} + objects := []runtime.Object{cephCluster} + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() + c, ready, clusterExists, _ := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.True(t, clusterExists) + }) + + t.Run("cephcluster with cleanup policy when deleted", func(t *testing.T) { + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName.Name, + Namespace: clusterName.Namespace, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + }, + Spec: cephv1.ClusterSpec{ + CleanupPolicy: cephv1.CleanupPolicySpec{ + Confirmation: cephv1.DeleteDataDirOnHostsConfirmation, + }, + }} + objects := []runtime.Object{cephCluster} + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() + c, ready, clusterExists, _ := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.False(t, clusterExists) + }) +} diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index ec77e79b3a00..528320147de8 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -185,7 +185,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteFilesystem() function since everything is gone already diff --git a/pkg/operator/ceph/file/mirror/controller.go b/pkg/operator/ceph/file/mirror/controller.go index a12d2f7fb4ff..9f1faaa9b794 100644 --- a/pkg/operator/ceph/file/mirror/controller.go +++ b/pkg/operator/ceph/file/mirror/controller.go @@ -158,7 +158,7 @@ func (r *ReconcileFilesystemMirror) reconcile(request reconcile.Request) (reconc } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { logger.Debugf("CephCluster resource not ready in namespace %q, retrying in %q.", request.NamespacedName.Namespace, reconcileResponse.RequeueAfter.String()) return reconcileResponse, nil diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index dd949c9053ee..0e57420a9fd3 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -168,7 +168,7 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteStore() function since everything is gone already diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index f62502a94370..737da3eb6866 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -184,7 +184,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteStore() function since everything is gone already diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 2e900b3be93a..5c179437344f 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -145,7 +145,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R } // Make sure a CephCluster is present otherwise do nothing - _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR if !cephObjectRealm.GetDeletionTimestamp().IsZero() && !cephClusterExists { diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index 8203cfa4f8b3..b387738c94d8 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -159,7 +159,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteUser() function since everything is gone already diff --git a/pkg/operator/ceph/object/zone/controller.go b/pkg/operator/ceph/object/zone/controller.go index f7948ee05ab3..6d625794272f 100644 --- a/pkg/operator/ceph/object/zone/controller.go +++ b/pkg/operator/ceph/object/zone/controller.go @@ -141,7 +141,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // diff --git a/pkg/operator/ceph/object/zonegroup/controller.go b/pkg/operator/ceph/object/zonegroup/controller.go index 310ddafe4ae0..6d2e22e2cc27 100644 --- a/pkg/operator/ceph/object/zonegroup/controller.go +++ b/pkg/operator/ceph/object/zonegroup/controller.go @@ -139,7 +139,7 @@ func (r *ReconcileObjectZoneGroup) reconcile(request reconcile.Request) (reconci } // Make sure a CephCluster is present otherwise do nothing - _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR if !cephObjectZoneGroup.GetDeletionTimestamp().IsZero() && !cephClusterExists { diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index 54bed2bc162f..154a21638398 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -165,7 +165,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deletePool() function since everything is gone already