ceph: retry object health check if creation fails

If the CephObjectStore health checker fails to be created, return a
reconcile failure so that the reconcile will be run again and Rook will
retry creating the health checker. This also means that Rook will not
list the CephObjectStore as ready if the health checker can't be
started.

Signed-off-by: Blaine Gardner <blaine.gardner@redhat.com>

pkg/operator/ceph/object/admin.go

@@ -150,7 +150,7 @@ func NewMultisiteAdminOpsContext(
 		return nil, errors.Wrapf(err, "failed to create or retrieve rgw admin ops user")
 	}
 
-	httpClient, tlsCert, err := GenObjectStoreHTTPClient(objContext, spec)
+	httpClient, tlsCert, err := genObjectStoreHTTPClientFunc(objContext, spec)
 	if err != nil {
 		return nil, err
 	}

pkg/operator/ceph/object/controller.go

@@ -448,7 +448,10 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
 
 	// Start monitoring
 	if !cephObjectStore.Spec.HealthCheck.Bucket.Disabled {
-		r.startMonitoring(cephObjectStore, objContext, namespacedName)
+		err = r.startMonitoring(cephObjectStore, objContext, namespacedName)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
 	}
 
 	return reconcile.Result{}, nil
@@ -513,20 +516,23 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1
 	return cephObjectStore.Name, cephObjectStore.Name, cephObjectStore.Name, reconcile.Result{}, nil
 }
 
-func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, namespacedName types.NamespacedName) {
+func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjectStore, objContext *Context, namespacedName types.NamespacedName) error {
 	// Start monitoring object store
 	if r.objectStoreContexts[objectstore.Name].started {
 		logger.Info("external rgw endpoint monitoring go routine already running!")
-		return
+		return nil
 	}
 
 	rgwChecker, err := newBucketChecker(r.context, objContext, r.client, namespacedName, &objectstore.Spec)
 	if err != nil {
-		logger.Error(err)
-		return
+		return errors.Wrapf(err, "failed to start rgw health checker for CephObjectStore %q, will re-reconcile", namespacedName.String())
 	}
 
-	logger.Info("starting rgw healthcheck")
+	logger.Infof("starting rgw health checker for CephObjectStore %q", namespacedName.String())
 	go rgwChecker.checkObjectStore(r.objectStoreContexts[objectstore.Name].internalCtx)
+
+	// Set the monitoring flag so we don't start more than one go routine
 	r.objectStoreContexts[objectstore.Name].started = true
+
+	return nil
 }