Merge pull request #8804 from jmolmo/fix_timeout
ceph: retry <ceph orch> commands when they fail
This commit is contained in:
1 file changed
+34
-11
@@ -33,6 +33,10 @@ import (
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTries = 3
|
||||
)
|
||||
|
||||
// **************************************************
|
||||
// *** Mgr operations covered by TestMgrSmokeSuite ***
|
||||
//
|
||||
@@ -93,7 +97,7 @@ func (s *CephMgrSuite) SetupSuite() {
|
||||
UsePVC: false,
|
||||
Mons: 1,
|
||||
SkipOSDCreation: true,
|
||||
EnableDiscovery: true,
|
||||
EnableDiscovery: false,
|
||||
RookVersion: installer.LocalBuildTag,
|
||||
CephVersion: installer.MasterVersion,
|
||||
}
|
||||
@@ -112,9 +116,28 @@ func (s *CephMgrSuite) TearDownSuite() {
|
||||
s.installer.UninstallRook()
|
||||
}
|
||||
|
||||
func (s *CephMgrSuite) execute(command []string) (error, string) {
|
||||
orchCommand := append([]string{"orch"}, command...)
|
||||
return s.installer.Execute("ceph", orchCommand, s.namespace)
|
||||
func (s *CephMgrSuite) executeWithRetry(command []string, maxRetries int) (string, error) {
|
||||
tries := 0
|
||||
orchestratorCommand := append([]string{"orch"}, command...)
|
||||
for {
|
||||
err, output := s.installer.Execute("ceph", orchestratorCommand, s.namespace)
|
||||
tries++
|
||||
if err != nil {
|
||||
if maxRetries == 1 {
|
||||
return output, err
|
||||
}
|
||||
if tries == maxRetries {
|
||||
return "", fmt.Errorf("max retries(%d) reached, last err: %v", tries, err)
|
||||
}
|
||||
logger.Infof("retrying command <<ceph %s>>: last error: %v", command, err)
|
||||
continue
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *CephMgrSuite) execute(command []string) (string, error) {
|
||||
return s.executeWithRetry(command, 1)
|
||||
}
|
||||
|
||||
func (s *CephMgrSuite) prepareLocalStorageClass(storageClassName string) {
|
||||
@@ -151,7 +174,7 @@ func (s *CephMgrSuite) enableOrchestratorModule() {
|
||||
}
|
||||
|
||||
logger.Info("Setting orchestrator backend to Rook .... <ceph orch set backend rook>")
|
||||
err, output = s.execute([]string{"set", "backend", "rook"})
|
||||
output, err = s.execute([]string{"set", "backend", "rook"})
|
||||
logger.Infof("output: %s", output)
|
||||
if err != nil {
|
||||
logger.Infof("Not possible to set rook as backend orchestrator module: %q", err)
|
||||
@@ -169,7 +192,7 @@ func (s *CephMgrSuite) waitForOrchestrationModule() {
|
||||
|
||||
for timeout := 0; timeout < 30; timeout++ {
|
||||
logger.Info("Waiting for rook orchestrator module enabled and ready ...")
|
||||
err, output := s.execute([]string{"status", "--format", "json"})
|
||||
output, err := s.execute([]string{"status", "--format", "json"})
|
||||
logger.Infof("%s", output)
|
||||
if err == nil {
|
||||
logger.Info("Ceph orchestrator ready to execute commands")
|
||||
@@ -209,14 +232,14 @@ func (s *CephMgrSuite) waitForOrchestrationModule() {
|
||||
}
|
||||
func (s *CephMgrSuite) TestDeviceLs() {
|
||||
logger.Info("Testing .... <ceph orch device ls>")
|
||||
err, device_list := s.execute([]string{"device", "ls"})
|
||||
deviceList, err := s.executeWithRetry([]string{"device", "ls"}, defaultTries)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", device_list)
|
||||
logger.Infof("output = %s", deviceList)
|
||||
}
|
||||
|
||||
func (s *CephMgrSuite) TestStatus() {
|
||||
logger.Info("Testing .... <ceph orch status>")
|
||||
err, status := s.execute([]string{"status"})
|
||||
status, err := s.executeWithRetry([]string{"status"}, defaultTries)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", status)
|
||||
|
||||
@@ -235,7 +258,7 @@ func (s *CephMgrSuite) TestHostLs() {
|
||||
logger.Info("Testing .... <ceph orch host ls>")
|
||||
|
||||
// Get the orchestrator hosts
|
||||
err, output := s.execute([]string{"host", "ls", "json"})
|
||||
output, err := s.executeWithRetry([]string{"host", "ls", "json"}, defaultTries)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", output)
|
||||
|
||||
@@ -270,7 +293,7 @@ func (s *CephMgrSuite) TestHostLs() {
|
||||
|
||||
func (s *CephMgrSuite) TestServiceLs() {
|
||||
logger.Info("Testing .... <ceph orch ls --format json>")
|
||||
err, output := s.execute([]string{"ls", "--format", "json"})
|
||||
output, err := s.executeWithRetry([]string{"ls", "--format", "json"}, defaultTries)
|
||||
assert.Nil(s.T(), err)
|
||||
logger.Infof("output = %s", output)
|
||||
|
||||
|
||||
Reference in new issue
Block a user