From 6b2501bed98ff74f0c24d475b2442d78fc881bed Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 26 Oct 2021 13:23:11 -0600 Subject: [PATCH] core: treat cluster as not existing if the cleanup policy is set The cluster CR can be forcefully deleted and cleanup the cluster resources if the yes-really-destroy-data policy is set on the CR. In this case, the other controllers should treat the cluster CR as not existing and allow the finalizers to be removed on those resources if they are requested for deletion. Signed-off-by: Travis Nielsen --- cluster/examples/kubernetes/ceph/cluster.yaml | 2 + pkg/operator/ceph/client/controller.go | 2 +- pkg/operator/ceph/cluster/rbd/controller.go | 2 +- .../ceph/controller/controller_utils.go | 11 ++- .../ceph/controller/controller_utils_test.go | 91 +++++++++++++++++++ pkg/operator/ceph/file/controller.go | 2 +- pkg/operator/ceph/file/mirror/controller.go | 2 +- pkg/operator/ceph/nfs/controller.go | 2 +- pkg/operator/ceph/object/controller.go | 2 +- pkg/operator/ceph/object/realm/controller.go | 2 +- pkg/operator/ceph/object/user/controller.go | 2 +- pkg/operator/ceph/object/zone/controller.go | 2 +- .../ceph/object/zonegroup/controller.go | 2 +- pkg/operator/ceph/pool/controller.go | 2 +- 14 files changed, 112 insertions(+), 14 deletions(-) diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 3a325a780e7df..4263c2045a803 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -114,6 +114,8 @@ spec: # To destroy all Rook data on hosts during uninstall, confirmation must be set to "yes-really-destroy-data". # This value should only be set when the cluster is about to be deleted. After the confirmation is set, # Rook will immediately stop configuring the cluster and only wait for the delete command. + # After the cleanup policy is set, if any other Ceph CRs (e.g. CephFilesystem, CephObjectStore, etc) + # are deleted, Rook will immediately remove their finalizers and allow them to be deleted. # If the empty string is set, Rook will not destroy any data on hosts during uninstall. confirmation: "" # sanitizeDisks represents settings for sanitizing OSD disks on cluster deletion diff --git a/pkg/operator/ceph/client/controller.go b/pkg/operator/ceph/client/controller.go index bd36e33c86d9a..1b2af01a472ea 100644 --- a/pkg/operator/ceph/client/controller.go +++ b/pkg/operator/ceph/client/controller.go @@ -152,7 +152,7 @@ func (r *ReconcileCephClient) reconcile(request reconcile.Request) (reconcile.Re } // Make sure a CephCluster is present otherwise do nothing - _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deletePool() function since everything is gone already diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index 8411cbf515085..2041bb0975f2c 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -172,7 +172,7 @@ func (r *ReconcileCephRBDMirror) reconcile(request reconcile.Request) (reconcile } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { logger.Debugf("CephCluster resource not ready in namespace %q, retrying in %q.", request.NamespacedName.Namespace, reconcileResponse.RequeueAfter.String()) return reconcileResponse, nil diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index ed399c761a78d..2bb5c6e1684ca 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -25,7 +25,6 @@ import ( "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -114,7 +113,7 @@ func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, control } // IsReadyToReconcile determines if a controller is ready to reconcile or not -func IsReadyToReconcile(ctx context.Context, c client.Client, clustercontext *clusterd.Context, namespacedName types.NamespacedName, controllerName string) (cephv1.CephCluster, bool, bool, reconcile.Result) { +func IsReadyToReconcile(ctx context.Context, c client.Client, namespacedName types.NamespacedName, controllerName string) (cephv1.CephCluster, bool, bool, reconcile.Result) { cephClusterExists := false // Running ceph commands won't work and the controller will keep re-queuing so I believe it's fine not to check @@ -130,9 +129,15 @@ func IsReadyToReconcile(ctx context.Context, c client.Client, clustercontext *cl logger.Debugf("%q: no CephCluster resource found in namespace %q", controllerName, namespacedName.Namespace) return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady } - cephClusterExists = true cephCluster = clusterList.Items[0] + // If the cluster has a cleanup policy to destroy the cluster and it has been marked for deletion, treat it as if it does not exist + if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.DeletionTimestamp.IsZero() { + logger.Infof("%q: CephCluster %q has a destructive cleanup policy, allowing resources to be deleted", controllerName, namespacedName) + return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady + } + + cephClusterExists = true logger.Debugf("%q: CephCluster resource %q found in namespace %q", controllerName, cephCluster.Name, namespacedName.Namespace) // read the CR status of the cluster diff --git a/pkg/operator/ceph/controller/controller_utils_test.go b/pkg/operator/ceph/controller/controller_utils_test.go index 52dac34a35bc8..d8872551084cf 100644 --- a/pkg/operator/ceph/controller/controller_utils_test.go +++ b/pkg/operator/ceph/controller/controller_utils_test.go @@ -17,12 +17,18 @@ limitations under the License. package controller import ( + ctx "context" "testing" "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/util/exec" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func CreateTestClusterFromStatusDetails(details map[string]cephv1.CephHealthMessage) cephv1.CephCluster { @@ -85,3 +91,88 @@ func TestSetCephCommandsTimeout(t *testing.T) { SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "1"}) assert.Equal(t, 1*time.Second, exec.CephCommandsTimeout) } + +func TestIsReadyToReconcile(t *testing.T) { + scheme := scheme.Scheme + scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) + + controllerName := "testing" + clusterName := types.NamespacedName{Name: "mycluster", Namespace: "myns"} + + t.Run("non-existent cephcluster", func(t *testing.T) { + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build() + c, ready, clusterExists, reconcileResult := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.False(t, clusterExists) + assert.Equal(t, WaitForRequeueIfCephClusterNotReady, reconcileResult) + }) + + t.Run("valid cephcluster", func(t *testing.T) { + cephCluster := &cephv1.CephCluster{} + objects := []runtime.Object{cephCluster} + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() + c, ready, clusterExists, reconcileResult := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.False(t, clusterExists) + assert.Equal(t, WaitForRequeueIfCephClusterNotReady, reconcileResult) + }) + + t.Run("deleted cephcluster with no cleanup policy", func(t *testing.T) { + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName.Name, + Namespace: clusterName.Namespace, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + }, + } + + objects := []runtime.Object{cephCluster} + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() + c, ready, clusterExists, reconcileResult := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.True(t, clusterExists) + assert.Equal(t, WaitForRequeueIfCephClusterNotReady, reconcileResult) + }) + + t.Run("cephcluster with cleanup policy when not deleted", func(t *testing.T) { + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName.Name, + Namespace: clusterName.Namespace, + }, + Spec: cephv1.ClusterSpec{ + CleanupPolicy: cephv1.CleanupPolicySpec{ + Confirmation: cephv1.DeleteDataDirOnHostsConfirmation, + }, + }} + objects := []runtime.Object{cephCluster} + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() + c, ready, clusterExists, _ := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.True(t, clusterExists) + }) + + t.Run("cephcluster with cleanup policy when deleted", func(t *testing.T) { + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName.Name, + Namespace: clusterName.Namespace, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + }, + Spec: cephv1.ClusterSpec{ + CleanupPolicy: cephv1.CleanupPolicySpec{ + Confirmation: cephv1.DeleteDataDirOnHostsConfirmation, + }, + }} + objects := []runtime.Object{cephCluster} + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() + c, ready, clusterExists, _ := IsReadyToReconcile(ctx.TODO(), client, clusterName, controllerName) + assert.NotNil(t, c) + assert.False(t, ready) + assert.False(t, clusterExists) + }) +} diff --git a/pkg/operator/ceph/file/controller.go b/pkg/operator/ceph/file/controller.go index ec77e79b3a000..528320147de8c 100644 --- a/pkg/operator/ceph/file/controller.go +++ b/pkg/operator/ceph/file/controller.go @@ -185,7 +185,7 @@ func (r *ReconcileCephFilesystem) reconcile(request reconcile.Request) (reconcil } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteFilesystem() function since everything is gone already diff --git a/pkg/operator/ceph/file/mirror/controller.go b/pkg/operator/ceph/file/mirror/controller.go index a12d2f7fb4ffe..9f1faaa9b7945 100644 --- a/pkg/operator/ceph/file/mirror/controller.go +++ b/pkg/operator/ceph/file/mirror/controller.go @@ -158,7 +158,7 @@ func (r *ReconcileFilesystemMirror) reconcile(request reconcile.Request) (reconc } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, _, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { logger.Debugf("CephCluster resource not ready in namespace %q, retrying in %q.", request.NamespacedName.Namespace, reconcileResponse.RequeueAfter.String()) return reconcileResponse, nil diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index dd949c9053ee9..0e57420a9fd33 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -168,7 +168,7 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteStore() function since everything is gone already diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index f62502a943709..737da3eb68665 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -184,7 +184,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteStore() function since everything is gone already diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index 2e900b3be93a6..5c179437344f1 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -145,7 +145,7 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R } // Make sure a CephCluster is present otherwise do nothing - _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR if !cephObjectRealm.GetDeletionTimestamp().IsZero() && !cephClusterExists { diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index 8203cfa4f8b36..b387738c94d8f 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -159,7 +159,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deleteUser() function since everything is gone already diff --git a/pkg/operator/ceph/object/zone/controller.go b/pkg/operator/ceph/object/zone/controller.go index f7948ee05ab3c..6d625794272fe 100644 --- a/pkg/operator/ceph/object/zone/controller.go +++ b/pkg/operator/ceph/object/zone/controller.go @@ -141,7 +141,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // diff --git a/pkg/operator/ceph/object/zonegroup/controller.go b/pkg/operator/ceph/object/zonegroup/controller.go index 310ddafe4ae0b..6d2e22e2cc278 100644 --- a/pkg/operator/ceph/object/zonegroup/controller.go +++ b/pkg/operator/ceph/object/zonegroup/controller.go @@ -139,7 +139,7 @@ func (r *ReconcileObjectZoneGroup) reconcile(request reconcile.Request) (reconci } // Make sure a CephCluster is present otherwise do nothing - _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + _, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR if !cephObjectZoneGroup.GetDeletionTimestamp().IsZero() && !cephClusterExists { diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index 54bed2bc162f5..154a21638398e 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -165,7 +165,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile } // Make sure a CephCluster is present otherwise do nothing - cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, r.context, request.NamespacedName, controllerName) + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { // This handles the case where the Ceph Cluster is gone and we want to delete that CR // We skip the deletePool() function since everything is gone already