Merge pull request #8804 from jmolmo/fix_timeout

ceph: retry <ceph orch> commands when they fail
This commit is contained in:
Blaine Gardner authored and GitHub committed 2021-10-01 10:55:32 -06:00
commit 08cb6785a4
1 file changed
+34 -11
+34 -11
View File
@@ -33,6 +33,10 @@ import (
"github.com/stretchr/testify/suite"
)
const (
defaultTries = 3
)
// **************************************************
// *** Mgr operations covered by TestMgrSmokeSuite ***
//
@@ -93,7 +97,7 @@ func (s *CephMgrSuite) SetupSuite() {
UsePVC: false,
Mons: 1,
SkipOSDCreation: true,
EnableDiscovery: true,
EnableDiscovery: false,
RookVersion: installer.LocalBuildTag,
CephVersion: installer.MasterVersion,
}
@@ -112,9 +116,28 @@ func (s *CephMgrSuite) TearDownSuite() {
s.installer.UninstallRook()
}
func (s *CephMgrSuite) execute(command []string) (error, string) {
orchCommand := append([]string{"orch"}, command...)
return s.installer.Execute("ceph", orchCommand, s.namespace)
func (s *CephMgrSuite) executeWithRetry(command []string, maxRetries int) (string, error) {
tries := 0
orchestratorCommand := append([]string{"orch"}, command...)
for {
err, output := s.installer.Execute("ceph", orchestratorCommand, s.namespace)
tries++
if err != nil {
if maxRetries == 1 {
return output, err
}
if tries == maxRetries {
return "", fmt.Errorf("max retries(%d) reached, last err: %v", tries, err)
}
logger.Infof("retrying command <<ceph %s>>: last error: %v", command, err)
continue
}
return output, nil
}
}
func (s *CephMgrSuite) execute(command []string) (string, error) {
return s.executeWithRetry(command, 1)
}
func (s *CephMgrSuite) prepareLocalStorageClass(storageClassName string) {
@@ -151,7 +174,7 @@ func (s *CephMgrSuite) enableOrchestratorModule() {
}
logger.Info("Setting orchestrator backend to Rook .... <ceph orch set backend rook>")
err, output = s.execute([]string{"set", "backend", "rook"})
output, err = s.execute([]string{"set", "backend", "rook"})
logger.Infof("output: %s", output)
if err != nil {
logger.Infof("Not possible to set rook as backend orchestrator module: %q", err)
@@ -169,7 +192,7 @@ func (s *CephMgrSuite) waitForOrchestrationModule() {
for timeout := 0; timeout < 30; timeout++ {
logger.Info("Waiting for rook orchestrator module enabled and ready ...")
err, output := s.execute([]string{"status", "--format", "json"})
output, err := s.execute([]string{"status", "--format", "json"})
logger.Infof("%s", output)
if err == nil {
logger.Info("Ceph orchestrator ready to execute commands")
@@ -209,14 +232,14 @@ func (s *CephMgrSuite) waitForOrchestrationModule() {
}
func (s *CephMgrSuite) TestDeviceLs() {
logger.Info("Testing .... <ceph orch device ls>")
err, device_list := s.execute([]string{"device", "ls"})
deviceList, err := s.executeWithRetry([]string{"device", "ls"}, defaultTries)
assert.Nil(s.T(), err)
logger.Infof("output = %s", device_list)
logger.Infof("output = %s", deviceList)
}
func (s *CephMgrSuite) TestStatus() {
logger.Info("Testing .... <ceph orch status>")
err, status := s.execute([]string{"status"})
status, err := s.executeWithRetry([]string{"status"}, defaultTries)
assert.Nil(s.T(), err)
logger.Infof("output = %s", status)
@@ -235,7 +258,7 @@ func (s *CephMgrSuite) TestHostLs() {
logger.Info("Testing .... <ceph orch host ls>")
// Get the orchestrator hosts
err, output := s.execute([]string{"host", "ls", "json"})
output, err := s.executeWithRetry([]string{"host", "ls", "json"}, defaultTries)
assert.Nil(s.T(), err)
logger.Infof("output = %s", output)
@@ -270,7 +293,7 @@ func (s *CephMgrSuite) TestHostLs() {
func (s *CephMgrSuite) TestServiceLs() {
logger.Info("Testing .... <ceph orch ls --format json>")
err, output := s.execute([]string{"ls", "--format", "json"})
output, err := s.executeWithRetry([]string{"ls", "--format", "json"}, defaultTries)
assert.Nil(s.T(), err)
logger.Infof("output = %s", output)