From 5c9bb7d9d7338f4575071dafcf3194a70eda68ad Mon Sep 17 00:00:00 2001
From: Noah Watkins <noahwatkins@gmail.com>
Date: Thu, 14 Mar 2019 14:50:54 -0700
Subject: [PATCH 1/3] ceph: add helper to execute cmd with retries

Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
---
 pkg/daemon/ceph/client/command.go | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/pkg/daemon/ceph/client/command.go b/pkg/daemon/ceph/client/command.go
index db8962a77..d16ed8ba6 100644
--- a/pkg/daemon/ceph/client/command.go
+++ b/pkg/daemon/ceph/client/command.go
@@ -141,3 +141,31 @@ func executeCommandWithOutputFile(context *clusterd.Context, debug bool, command
 	output, err := context.Executor.ExecuteCommandWithOutputFile(debug, "", command, "--out-file", args...)
 	return []byte(output), err
 }
+
+func ExecuteCephCommandWithRetry(
+	cmd func() ([]byte, error),
+	getExitCode func(err error) (int, bool),
+	retries int,
+	retryOnExitCode int,
+	waitTime time.Duration,
+) ([]byte, error) {
+	for i := 0; i < retries; i++ {
+		data, err := cmd()
+		if err != nil {
+			exitCode, parsed := getExitCode(err)
+			if parsed {
+				if exitCode == retryOnExitCode {
+					logger.Infof("command failed. trying again...")
+					time.Sleep(waitTime)
+					continue
+				}
+			}
+			return nil, fmt.Errorf("failed to complete command %+v", err)
+		}
+		if i > 0 {
+			logger.Infof("command succeeded on attempt %d", i)
+		}
+		return data, nil
+	}
+	return nil, fmt.Errorf("max command retries exceeded")
+}

From 8ebf9ac722ceffc5a8382fb965a33629b6954440 Mon Sep 17 00:00:00 2001
From: Noah Watkins <noahwatkins@gmail.com>
Date: Thu, 14 Mar 2019 14:51:50 -0700
Subject: [PATCH 2/3] ceph: retry dashboard credentials setup

may need retry if manager hasn't fully loaded its modules.

fixes: #2809

Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
---
 pkg/operator/ceph/cluster/mgr/dashboard.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go
index 29d3592c4..3d85672c5 100755
--- a/pkg/operator/ceph/cluster/mgr/dashboard.go
+++ b/pkg/operator/ceph/cluster/mgr/dashboard.go
@@ -220,8 +220,11 @@ func getExitCode(err error) (int, bool) {
 func (c *Cluster) setLoginCredentials(password string) error {
 	// Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log.
 	logger.Infof("Running command: ceph dashboard set-login-credentials admin *******")
-	args := []string{"dashboard", "set-login-credentials", dashboardUsername, password}
-	_, err := client.ExecuteCephCommandDebugLog(c.context, c.Namespace, args)
+	// retry a few times in the case that the mgr module is not ready to accept commands
+	_, err := client.ExecuteCephCommandWithRetry(func() ([]byte, error) {
+		args := []string{"dashboard", "set-login-credentials", dashboardUsername, password}
+		return client.ExecuteCephCommandDebugLog(c.context, c.Namespace, args)
+	}, c.exitCode, 5, invalidArgErrorCode, dashboardInitWaitTime)
 	if err != nil {
 		return fmt.Errorf("failed to set login creds on mgr. %+v", err)
 	}

From 1ee7cdfa8b3b2315138f18ad692b97fa92344855 Mon Sep 17 00:00:00 2001
From: Noah Watkins <noahwatkins@gmail.com>
Date: Thu, 14 Mar 2019 14:52:44 -0700
Subject: [PATCH 3/3] ceph: refactor dashboard retry logic to use helper

Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
---
 pkg/operator/ceph/cluster/mgr/orchestrator.go | 21 +++++++------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/pkg/operator/ceph/cluster/mgr/orchestrator.go b/pkg/operator/ceph/cluster/mgr/orchestrator.go
index a1d1e0471..a3ad0443b 100644
--- a/pkg/operator/ceph/cluster/mgr/orchestrator.go
+++ b/pkg/operator/ceph/cluster/mgr/orchestrator.go
@@ -48,20 +48,13 @@ func (c *Cluster) configureOrchestratorModules() error {
 		return fmt.Errorf("failed to enable mgr rook module. %+v", err)
 	}
 	// retry a few times in the case that the mgr module is not ready to accept commands
-	for i := 0; i < 5; i++ {
-		_, err := client.ExecuteCephCommand(c.context, c.Namespace, []string{"orchestrator", "set", "backend", "rook"})
-		if err != nil {
-			exitCode, parsed := c.exitCode(err)
-			if parsed {
-				if exitCode == invalidArgErrorCode {
-					logger.Infof("orchestrator module is not ready yet. trying again...")
-					time.Sleep(orchestratorInitWaitTime)
-					continue
-				}
-			}
-			return fmt.Errorf("failed to set rook as the orchestrator backend. %+v", err)
-		}
-		break
+	_, err := client.ExecuteCephCommandWithRetry(func() ([]byte, error) {
+		args := []string{"orchestrator", "set", "backend", "rook"}
+		return client.ExecuteCephCommand(c.context, c.Namespace, args)
+	}, c.exitCode, 5, invalidArgErrorCode, orchestratorInitWaitTime)
+	if err != nil {
+		return fmt.Errorf("failed to set rook as the orchestrator backend. %+v", err)
+
 	}
 
 	return nil