Merge pull request #2414 from travisn/retry-dashboard-cert

Retry creating self signed cert if module is not ready

pkg/operator/ceph/cluster/mgr/dashboard.go

@@ -41,6 +41,7 @@ const (
 	passwordLength                 = 10
 	passwordKeyName                = "password"
 	certAlreadyConfiguredErrorCode = 5
+	invalidArgErrorCode            = 22
 )
 
 var (
@@ -145,22 +146,40 @@ func (c *Cluster) initializeSecureDashboard() error {
 func (c *Cluster) createSelfSignedCert() (bool, error) {
 	// create a self-signed cert for the https connections required in mimic
 	args := []string{"dashboard", "create-self-signed-cert"}
-	_, err := client.ExecuteCephCommand(c.context, c.Namespace, args)
-	if err != nil {
-		if exiterr, ok := err.(*exec.ExitError); ok {
-			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
-				exitCode := status.ExitStatus()
+
+	// retry a few times in the case that the mgr module is not ready to accept commands
+	for i := 0; i < 5; i++ {
+		_, err := client.ExecuteCephCommand(c.context, c.Namespace, args)
+		if err != nil {
+			exitCode, parsed := c.exitCode(err)
+			if parsed {
 				if exitCode == certAlreadyConfiguredErrorCode {
 					logger.Infof("dashboard is already initialized with a cert")
 					return true, nil
 				}
+				if exitCode == invalidArgErrorCode {
+					logger.Infof("dashboard module is not ready yet. trying again...")
+					time.Sleep(dashboardInitWaitTime)
+					continue
+				}
 			}
+			return false, fmt.Errorf("failed to create self signed cert on mgr. %+v", err)
 		}
-		return false, fmt.Errorf("failed to create self signed cert on mgr. %+v", err)
+		break
 	}
 	return false, nil
 }
 
+// Get the return code from the process
+func getExitCode(err error) (int, bool) {
+	if exiterr, ok := err.(*exec.ExitError); ok {
+		if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+			return status.ExitStatus(), true
+		}
+	}
+	return 0, false
+}
+
 func (c *Cluster) setLoginCredentials(password string) error {
 	// Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log.
 	logger.Infof("Running command: ceph dashboard set-login-credentials admin *******")

pkg/operator/ceph/cluster/mgr/dashboard_test.go

@@ -16,6 +16,7 @@ limitations under the License.
 package mgr
 
 import (
+	"fmt"
 	"testing"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -66,9 +67,12 @@ func TestGetOrGeneratePassword(t *testing.T) {
 func TestStartSecureDashboard(t *testing.T) {
 	enables := 0
 	disables := 0
+	moduleRetries := 0
+	exitCodeResponse := 0
 	executor := &exectest.MockExecutor{
 		MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) {
 			logger.Infof("command: %s %v", command, args)
+			exitCodeResponse = 0
 			if args[1] == "module" {
 				if args[2] == "enable" {
 					enables++
@@ -76,17 +80,33 @@ func TestStartSecureDashboard(t *testing.T) {
 					disables++
 				}
 			}
+			if args[0] == "dashboard" && args[1] == "create-self-signed-cert" {
+				if moduleRetries < 2 {
+					logger.Infof("simulating retry...")
+					exitCodeResponse = invalidArgErrorCode
+					moduleRetries++
+					return "", fmt.Errorf("test failure")
+				}
+			}
 			return "", nil
 		},
 	}
 	c := &Cluster{context: &clusterd.Context{Clientset: test.New(3), Executor: executor}, Namespace: "myns",
 		dashboard: cephv1.DashboardSpec{Enabled: true}, cephVersion: cephv1.CephVersionSpec{Name: cephv1.Mimic, Image: "ceph/ceph:v13.2.2"}}
+	c.exitCode = func(err error) (int, bool) {
+		if exitCodeResponse != 0 {
+			return exitCodeResponse, true
+		}
+		return exitCodeResponse, false
+	}
+
 	dashboardInitWaitTime = 0
 	err := c.configureDashboard(dashboardPortHttp)
 	assert.Nil(t, err)
 	// the dashboard is enabled, then disabled and enabled again to restart it with the cert
 	assert.Equal(t, 2, enables)
 	assert.Equal(t, 1, disables)
+	assert.Equal(t, 2, moduleRetries)
 
 	svc, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get("rook-ceph-mgr-dashboard", metav1.GetOptions{})
 	assert.Nil(t, err)

pkg/operator/ceph/cluster/mgr/mgr.go

@@ -56,6 +56,7 @@ type Cluster struct {
 	dashboard   cephv1.DashboardSpec
 	cephVersion cephv1.CephVersionSpec
 	rookVersion string
+	exitCode    func(err error) (int, bool)
 }
 
 // mgrConfig for a single mgr
@@ -79,6 +80,7 @@ func New(context *clusterd.Context, namespace, rookVersion string, cephVersion c
 		HostNetwork: hostNetwork,
 		resources:   resources,
 		ownerRef:    ownerRef,
+		exitCode:    getExitCode,
 	}
 }