From 464e332dc1849c9d3bc5d1e70eba0cfb2eab0313 Mon Sep 17 00:00:00 2001
From: Travis Nielsen <tnielsen@redhat.com>
Date: Wed, 6 Jan 2021 17:08:15 -0700
Subject: [PATCH] ceph: init rgw dashboard access key in goroutine

The command to set or disable the rgw dashboard started
hanging in some scenarios in v15.2.8. For now we start
the rgw dashboard config in a goroutine until this issue
is tracked down. After the issue is fixed in ceph, the
goroutines will no longer be necessary.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
---
 pkg/operator/ceph/object/objectstore.go      | 60 +++++++++++---------
 pkg/operator/ceph/object/objectstore_test.go |  3 +-
 pkg/operator/ceph/object/rgw.go              |  5 +-
 3 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go
index 19ba7f423..f73c4e4e1 100644
--- a/pkg/operator/ceph/object/objectstore.go
+++ b/pkg/operator/ceph/object/objectstore.go
@@ -23,6 +23,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -46,6 +47,9 @@ const (
 	AccessKeyName         = "access-key"
 	SecretKeyName         = "secret-key"
 	svcDNSSuffix          = "svc"
+
+	// Timeout for setting the dashboard access key
+	applyDashboardKeyTimeout = 15 * time.Second
 )
 
 var (
@@ -730,6 +734,7 @@ func checkDashboardUser(context *Context) (bool, error) {
 }
 
 func enableRGWDashboard(context *Context) error {
+	logger.Info("enabling rgw dashboard")
 	checkDashboard, err := checkDashboardUser(context)
 	if err != nil {
 		logger.Debug("Unable to fetch dashboard user key for RGW, hence skipping")
@@ -757,47 +762,46 @@ func enableRGWDashboard(context *Context) error {
 
 	args = []string{"dashboard", "set-rgw-api-secret-key", *u.SecretKey}
 	cephCmd = ceph.NewCephCommand(context.Context, context.clusterInfo, args)
-	_, err = cephCmd.Run()
-	if err != nil {
-		return errors.Wrapf(err, "failed to set user %q secretkey", DashboardUser)
-	}
-
+	go func() {
+		// Setting the dashboard api secret started hanging in some clusters
+		// starting in ceph v15.2.8. We run it in a goroutine until the fix
+		// is found. We expect the ceph command to timeout so at least the goroutine exits.
+		logger.Info("setting the dashboard api secret key")
+		_, err = cephCmd.RunWithTimeout(applyDashboardKeyTimeout)
+		if err != nil {
+			logger.Errorf("failed to set user %q secretkey. %v", DashboardUser, err)
+		}
+		logger.Info("done setting the dashboard api secret key")
+	}()
 	return nil
 }
 
-func disableRGWDashboard(context *Context) error {
-	checkDashboard, err := checkDashboardUser(context)
+func disableRGWDashboard(context *Context) {
+	logger.Info("disabling the dashboard api user and secret key")
+
+	_, _, err := GetUser(context, DashboardUser)
 	if err != nil {
-		logger.Debugf("unable to fetch dashboard user key for user %q, hence skipping", DashboardUser)
-		return nil
-	}
-	if !checkDashboard {
-		logger.Debug("RGW Dashboard is already disabled")
-		return nil
-	}
-	_, _, err = GetUser(context, DashboardUser)
-	if err != nil {
-		logger.Debugf("unable to fetch the user %q details from this objectstore %q", DashboardUser, context.Name)
-		return nil
-	}
-	_, err = DeleteUser(context, DashboardUser)
-	if err != nil {
-		return errors.Wrapf(err, "failed to delete ceph user %q", DashboardUser)
+		logger.Infof("unable to fetch the user %q details from this objectstore %q", DashboardUser, context.Name)
+	} else {
+		logger.Info("deleting rgw dashboard user")
+		_, err = DeleteUser(context, DashboardUser)
+		if err != nil {
+			logger.Warningf("failed to delete ceph user %q. %v", DashboardUser, err)
+		}
 	}
 
 	args := []string{"dashboard", "reset-rgw-api-access-key"}
 	cephCmd := ceph.NewCephCommand(context.Context, context.clusterInfo, args)
-	_, err = cephCmd.Run()
+	_, err = cephCmd.RunWithTimeout(applyDashboardKeyTimeout)
 	if err != nil {
-		return errors.Wrapf(err, "failed to reset user accesskey for user %q", DashboardUser)
+		logger.Warningf("failed to reset user accesskey for user %q. %v", DashboardUser, err)
 	}
 
 	args = []string{"dashboard", "reset-rgw-api-secret-key"}
 	cephCmd = ceph.NewCephCommand(context.Context, context.clusterInfo, args)
-	_, err = cephCmd.Run()
+	_, err = cephCmd.RunWithTimeout(applyDashboardKeyTimeout)
 	if err != nil {
-		return errors.Wrapf(err, "failed to reset user secretkey for user %q", DashboardUser)
+		logger.Warningf("failed to reset user secretkey for user %q. %v", DashboardUser, err)
 	}
-
-	return nil
+	logger.Info("done disabling the dashboard api secret key")
 }
diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go
index 5d198c44a..893d712cf 100644
--- a/pkg/operator/ceph/object/objectstore_test.go
+++ b/pkg/operator/ceph/object/objectstore_test.go
@@ -261,6 +261,5 @@ func TestDashboard(t *testing.T) {
 	checkdashboard, err = checkDashboardUser(objContext)
 	assert.NoError(t, err)
 	assert.True(t, checkdashboard)
-	err = disableRGWDashboard(objContext)
-	assert.Nil(t, err)
+	disableRGWDashboard(objContext)
 }
diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go
index 669380443..6ca89eb96 100644
--- a/pkg/operator/ceph/object/rgw.go
+++ b/pkg/operator/ceph/object/rgw.go
@@ -241,10 +241,7 @@ func (c *clusterConfig) deleteStore() error {
 
 		objContext.Endpoint = c.store.Status.Info["endpoint"]
 
-		err = disableRGWDashboard(objContext)
-		if err != nil {
-			logger.Warningf("failed to disable dashboard for rgw. %v", err)
-		}
+		go disableRGWDashboard(objContext)
 
 		err = deleteRealmAndPools(objContext, c.store.Spec)
 		if err != nil {