diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go index 639db7264..e1814bc94 100755 --- a/pkg/operator/ceph/cluster/mgr/dashboard.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -41,6 +41,7 @@ const ( passwordLength = 10 passwordKeyName = "password" certAlreadyConfiguredErrorCode = 5 + invalidArgErrorCode = 22 ) var ( @@ -145,22 +146,40 @@ func (c *Cluster) initializeSecureDashboard() error { func (c *Cluster) createSelfSignedCert() (bool, error) { // create a self-signed cert for the https connections required in mimic args := []string{"dashboard", "create-self-signed-cert"} - _, err := client.ExecuteCephCommand(c.context, c.Namespace, args) - if err != nil { - if exiterr, ok := err.(*exec.ExitError); ok { - if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { - exitCode := status.ExitStatus() + + // retry a few times in the case that the mgr module is not ready to accept commands + for i := 0; i < 5; i++ { + _, err := client.ExecuteCephCommand(c.context, c.Namespace, args) + if err != nil { + exitCode, parsed := c.exitCode(err) + if parsed { if exitCode == certAlreadyConfiguredErrorCode { logger.Infof("dashboard is already initialized with a cert") return true, nil } + if exitCode == invalidArgErrorCode { + logger.Infof("dashboard module is not ready yet. trying again...") + time.Sleep(dashboardInitWaitTime) + continue + } } + return false, fmt.Errorf("failed to create self signed cert on mgr. %+v", err) } - return false, fmt.Errorf("failed to create self signed cert on mgr. %+v", err) + break } return false, nil } +// Get the return code from the process +func getExitCode(err error) (int, bool) { + if exiterr, ok := err.(*exec.ExitError); ok { + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + return status.ExitStatus(), true + } + } + return 0, false +} + func (c *Cluster) setLoginCredentials(password string) error { // Set the login credentials. Write the command/args to the debug log so we don't write the password by default to the log. logger.Infof("Running command: ceph dashboard set-login-credentials admin *******") diff --git a/pkg/operator/ceph/cluster/mgr/dashboard_test.go b/pkg/operator/ceph/cluster/mgr/dashboard_test.go index c38a31eba..0505c8f18 100644 --- a/pkg/operator/ceph/cluster/mgr/dashboard_test.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard_test.go @@ -16,6 +16,7 @@ limitations under the License. package mgr import ( + "fmt" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -66,9 +67,12 @@ func TestGetOrGeneratePassword(t *testing.T) { func TestStartSecureDashboard(t *testing.T) { enables := 0 disables := 0 + moduleRetries := 0 + exitCodeResponse := 0 executor := &exectest.MockExecutor{ MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { logger.Infof("command: %s %v", command, args) + exitCodeResponse = 0 if args[1] == "module" { if args[2] == "enable" { enables++ @@ -76,17 +80,33 @@ func TestStartSecureDashboard(t *testing.T) { disables++ } } + if args[0] == "dashboard" && args[1] == "create-self-signed-cert" { + if moduleRetries < 2 { + logger.Infof("simulating retry...") + exitCodeResponse = invalidArgErrorCode + moduleRetries++ + return "", fmt.Errorf("test failure") + } + } return "", nil }, } c := &Cluster{context: &clusterd.Context{Clientset: test.New(3), Executor: executor}, Namespace: "myns", dashboard: cephv1.DashboardSpec{Enabled: true}, cephVersion: cephv1.CephVersionSpec{Name: cephv1.Mimic, Image: "ceph/ceph:v13.2.2"}} + c.exitCode = func(err error) (int, bool) { + if exitCodeResponse != 0 { + return exitCodeResponse, true + } + return exitCodeResponse, false + } + dashboardInitWaitTime = 0 err := c.configureDashboard(dashboardPortHttp) assert.Nil(t, err) // the dashboard is enabled, then disabled and enabled again to restart it with the cert assert.Equal(t, 2, enables) assert.Equal(t, 1, disables) + assert.Equal(t, 2, moduleRetries) svc, err := c.context.Clientset.CoreV1().Services(c.Namespace).Get("rook-ceph-mgr-dashboard", metav1.GetOptions{}) assert.Nil(t, err) diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 33bbdc84f..1b3f4ea1f 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -56,6 +56,7 @@ type Cluster struct { dashboard cephv1.DashboardSpec cephVersion cephv1.CephVersionSpec rookVersion string + exitCode func(err error) (int, bool) } // mgrConfig for a single mgr @@ -79,6 +80,7 @@ func New(context *clusterd.Context, namespace, rookVersion string, cephVersion c HostNetwork: hostNetwork, resources: resources, ownerRef: ownerRef, + exitCode: getExitCode, } }