diff --git a/tests/integration/ceph_mgr_test.go b/tests/integration/ceph_mgr_test.go index d4238262b318..940df59c0acf 100644 --- a/tests/integration/ceph_mgr_test.go +++ b/tests/integration/ceph_mgr_test.go @@ -33,6 +33,10 @@ import ( "github.com/stretchr/testify/suite" ) +const ( + defaultTries = 3 +) + // ************************************************** // *** Mgr operations covered by TestMgrSmokeSuite *** // @@ -93,7 +97,7 @@ func (s *CephMgrSuite) SetupSuite() { UsePVC: false, Mons: 1, SkipOSDCreation: true, - EnableDiscovery: true, + EnableDiscovery: false, RookVersion: installer.LocalBuildTag, CephVersion: installer.MasterVersion, } @@ -112,9 +116,28 @@ func (s *CephMgrSuite) TearDownSuite() { s.installer.UninstallRook() } -func (s *CephMgrSuite) execute(command []string) (error, string) { - orchCommand := append([]string{"orch"}, command...) - return s.installer.Execute("ceph", orchCommand, s.namespace) +func (s *CephMgrSuite) executeWithRetry(command []string, maxRetries int) (string, error) { + tries := 0 + orchestratorCommand := append([]string{"orch"}, command...) + for { + err, output := s.installer.Execute("ceph", orchestratorCommand, s.namespace) + tries++ + if err != nil { + if maxRetries == 1 { + return output, err + } + if tries == maxRetries { + return "", fmt.Errorf("max retries(%d) reached, last err: %v", tries, err) + } + logger.Infof("retrying command <>: last error: %v", command, err) + continue + } + return output, nil + } +} + +func (s *CephMgrSuite) execute(command []string) (string, error) { + return s.executeWithRetry(command, 1) } func (s *CephMgrSuite) prepareLocalStorageClass(storageClassName string) { @@ -151,7 +174,7 @@ func (s *CephMgrSuite) enableOrchestratorModule() { } logger.Info("Setting orchestrator backend to Rook .... ") - err, output = s.execute([]string{"set", "backend", "rook"}) + output, err = s.execute([]string{"set", "backend", "rook"}) logger.Infof("output: %s", output) if err != nil { logger.Infof("Not possible to set rook as backend orchestrator module: %q", err) @@ -169,7 +192,7 @@ func (s *CephMgrSuite) waitForOrchestrationModule() { for timeout := 0; timeout < 30; timeout++ { logger.Info("Waiting for rook orchestrator module enabled and ready ...") - err, output := s.execute([]string{"status", "--format", "json"}) + output, err := s.execute([]string{"status", "--format", "json"}) logger.Infof("%s", output) if err == nil { logger.Info("Ceph orchestrator ready to execute commands") @@ -209,14 +232,14 @@ func (s *CephMgrSuite) waitForOrchestrationModule() { } func (s *CephMgrSuite) TestDeviceLs() { logger.Info("Testing .... ") - err, device_list := s.execute([]string{"device", "ls"}) + deviceList, err := s.executeWithRetry([]string{"device", "ls"}, defaultTries) assert.Nil(s.T(), err) - logger.Infof("output = %s", device_list) + logger.Infof("output = %s", deviceList) } func (s *CephMgrSuite) TestStatus() { logger.Info("Testing .... ") - err, status := s.execute([]string{"status"}) + status, err := s.executeWithRetry([]string{"status"}, defaultTries) assert.Nil(s.T(), err) logger.Infof("output = %s", status) @@ -235,7 +258,7 @@ func (s *CephMgrSuite) TestHostLs() { logger.Info("Testing .... ") // Get the orchestrator hosts - err, output := s.execute([]string{"host", "ls", "json"}) + output, err := s.executeWithRetry([]string{"host", "ls", "json"}, defaultTries) assert.Nil(s.T(), err) logger.Infof("output = %s", output) @@ -270,7 +293,7 @@ func (s *CephMgrSuite) TestHostLs() { func (s *CephMgrSuite) TestServiceLs() { logger.Info("Testing .... ") - err, output := s.execute([]string{"ls", "--format", "json"}) + output, err := s.executeWithRetry([]string{"ls", "--format", "json"}, defaultTries) assert.Nil(s.T(), err) logger.Infof("output = %s", output)