ceph: retry <ceph orch> commands when they fail

fixes: #8759

Signed-off-by: Juan Miguel Olmo Martínez <jolmomar@redhat.com>

tests/integration/ceph_mgr_test.go

@@ -33,6 +33,10 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
+const (
+	defaultTries = 3
+)
+
 // **************************************************
 // *** Mgr operations covered by TestMgrSmokeSuite ***
 //
@@ -94,7 +98,7 @@ func (s *CephMgrSuite) SetupSuite() {
 		Mons:              1,
 		UseCSI:            true,
 		SkipOSDCreation:   true,
-		EnableDiscovery:   true,
+		EnableDiscovery:   false,
 		RookVersion:       installer.LocalBuildTag,
 		CephVersion:       installer.MasterVersion,
 	}
@@ -113,9 +117,28 @@ func (s *CephMgrSuite) TearDownSuite() {
 	s.installer.UninstallRook()
 }
 
-func (s *CephMgrSuite) execute(command []string) (error, string) {
-	orchCommand := append([]string{"orch"}, command...)
-	return s.installer.Execute("ceph", orchCommand, s.namespace)
+func (s *CephMgrSuite) executeWithRetry(command []string, maxRetries int) (string, error) {
+	tries := 0
+	orchestratorCommand := append([]string{"orch"}, command...)
+	for {
+		err, output := s.installer.Execute("ceph", orchestratorCommand, s.namespace)
+		tries++
+		if err != nil  {
+			if maxRetries == 1 {
+				return output, err
+			}
+			if tries == maxRetries {
+				return "", fmt.Errorf("max retries(%d) reached, last err: %v", tries, err)
+			}
+			logger.Infof("retrying command <<ceph %s>>: last error: %v", command, err)
+			continue
+		}
+		return output, nil
+	}
+}
+
+func (s *CephMgrSuite) execute(command []string) (string, error) {
+	return s.executeWithRetry(command, 1)
 }
 
 func (s *CephMgrSuite) prepareLocalStorageClass(storageClassName string) {
@@ -152,7 +175,7 @@ func (s *CephMgrSuite) enableOrchestratorModule() {
 	}
 
 	logger.Info("Setting orchestrator backend to Rook .... <ceph orch set backend rook>")
-	err, output = s.execute([]string{"set", "backend", "rook"})
+	output, err = s.execute([]string{"set", "backend", "rook"})
 	logger.Infof("output: %s", output)
 	if err != nil {
 		logger.Infof("Not possible to set rook as backend orchestrator module: %q", err)
@@ -170,7 +193,7 @@ func (s *CephMgrSuite) waitForOrchestrationModule() {
 
 	for timeout := 0; timeout < 30; timeout++ {
 		logger.Info("Waiting for rook orchestrator module enabled and ready ...")
-		err, output := s.execute([]string{"status", "--format", "json"})
+		output, err := s.execute([]string{"status", "--format", "json"})
 		logger.Infof("%s", output)
 		if err == nil {
 			logger.Info("Ceph orchestrator ready to execute commands")
@@ -210,14 +233,14 @@ func (s *CephMgrSuite) waitForOrchestrationModule() {
 }
 func (s *CephMgrSuite) TestDeviceLs() {
 	logger.Info("Testing .... <ceph orch device ls>")
-	err, device_list := s.execute([]string{"device", "ls"})
+	deviceList, err := s.executeWithRetry([]string{"device", "ls"}, defaultTries)
 	assert.Nil(s.T(), err)
-	logger.Infof("output = %s", device_list)
+	logger.Infof("output = %s", deviceList)
 }
 
 func (s *CephMgrSuite) TestStatus() {
 	logger.Info("Testing .... <ceph orch status>")
-	err, status := s.execute([]string{"status"})
+	status, err := s.executeWithRetry([]string{"status"}, defaultTries)
 	assert.Nil(s.T(), err)
 	logger.Infof("output = %s", status)
 
@@ -236,7 +259,7 @@ func (s *CephMgrSuite) TestHostLs() {
 	logger.Info("Testing .... <ceph orch host ls>")
 
 	// Get the orchestrator hosts
-	err, output := s.execute([]string{"host", "ls", "json"})
+	output, err := s.executeWithRetry([]string{"host", "ls", "json"}, defaultTries)
 	assert.Nil(s.T(), err)
 	logger.Infof("output = %s", output)
 
@@ -271,7 +294,7 @@ func (s *CephMgrSuite) TestHostLs() {
 
 func (s *CephMgrSuite) TestServiceLs() {
 	logger.Info("Testing .... <ceph orch ls --format json>")
-	err, output := s.execute([]string{"ls", "--format", "json"})
+	output, err := s.executeWithRetry([]string{"ls", "--format", "json"}, defaultTries)
 	assert.Nil(s.T(), err)
 	logger.Infof("output = %s", output)