Merge pull request #2829 from noahdesu/dashboard-retry

retry dashboard operations

pkg/daemon/ceph/client/command.go

@@ -141,3 +141,31 @@ func executeCommandWithOutputFile(context *clusterd.Context, debug bool, command
 	output, err := context.Executor.ExecuteCommandWithOutputFile(debug, "", command, "--out-file", args...)
 	return []byte(output), err
 }
+
+func ExecuteCephCommandWithRetry(
+	cmd func() ([]byte, error),
+	getExitCode func(err error) (int, bool),
+	retries int,
+	retryOnExitCode int,
+	waitTime time.Duration,
+) ([]byte, error) {
+	for i := 0; i < retries; i++ {
+		data, err := cmd()
+		if err != nil {
+			exitCode, parsed := getExitCode(err)
+			if parsed {
+				if exitCode == retryOnExitCode {
+					logger.Infof("command failed. trying again...")
+					time.Sleep(waitTime)
+					continue
+				}
+			}
+			return nil, fmt.Errorf("failed to complete command %+v", err)
+		}
+		if i > 0 {
+			logger.Infof("command succeeded on attempt %d", i)
+		}
+		return data, nil
+	}
+	return nil, fmt.Errorf("max command retries exceeded")
+}

pkg/operator/ceph/cluster/mgr/dashboard.go

@@ -220,8 +220,11 @@ func getExitCode(err error) (int, bool) {
 func (c *Cluster) setLoginCredentials(password string) error {
 	// Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log.
 	logger.Infof("Running command: ceph dashboard set-login-credentials admin *******")
-	args := []string{"dashboard", "set-login-credentials", dashboardUsername, password}
-	_, err := client.ExecuteCephCommandDebugLog(c.context, c.Namespace, args)
+	// retry a few times in the case that the mgr module is not ready to accept commands
+	_, err := client.ExecuteCephCommandWithRetry(func() ([]byte, error) {
+		args := []string{"dashboard", "set-login-credentials", dashboardUsername, password}
+		return client.ExecuteCephCommandDebugLog(c.context, c.Namespace, args)
+	}, c.exitCode, 5, invalidArgErrorCode, dashboardInitWaitTime)
 	if err != nil {
 		return fmt.Errorf("failed to set login creds on mgr. %+v", err)
 	}

pkg/operator/ceph/cluster/mgr/orchestrator.go

@@ -48,20 +48,13 @@ func (c *Cluster) configureOrchestratorModules() error {
 		return fmt.Errorf("failed to enable mgr rook module. %+v", err)
 	}
 	// retry a few times in the case that the mgr module is not ready to accept commands
-	for i := 0; i < 5; i++ {
-		_, err := client.ExecuteCephCommand(c.context, c.Namespace, []string{"orchestrator", "set", "backend", "rook"})
-		if err != nil {
-			exitCode, parsed := c.exitCode(err)
-			if parsed {
-				if exitCode == invalidArgErrorCode {
-					logger.Infof("orchestrator module is not ready yet. trying again...")
-					time.Sleep(orchestratorInitWaitTime)
-					continue
-				}
-			}
-			return fmt.Errorf("failed to set rook as the orchestrator backend. %+v", err)
-		}
-		break
+	_, err := client.ExecuteCephCommandWithRetry(func() ([]byte, error) {
+		args := []string{"orchestrator", "set", "backend", "rook"}
+		return client.ExecuteCephCommand(c.context, c.Namespace, args)
+	}, c.exitCode, 5, invalidArgErrorCode, orchestratorInitWaitTime)
+	if err != nil {
+		return fmt.Errorf("failed to set rook as the orchestrator backend. %+v", err)
+
 	}
 
 	return nil