Merge pull request #5490 from travisn/monitoring-goroutines

Only run the cluster monitoring goroutines once
This commit is contained in:
Sébastien Han authored and GitHub committed 2020-05-19 09:40:13 +02:00
commit d28e2f605c
7 files changed
+79 -76

No files matched your search

+1 -2
View File
@@ -22,7 +22,6 @@ import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
)
type OSDUsage struct {
@@ -253,7 +252,7 @@ func OSDOut(context *clusterd.Context, clusterName string, osdID int) (string, e
return string(buf), err
}
func OsdSafeToDestroy(context *clusterd.Context, clusterName string, osdID int, cephVersion cephver.CephVersion) (bool, error) {
func OsdSafeToDestroy(context *clusterd.Context, clusterName string, osdID int) (bool, error) {
args := []string{"osd", "safe-to-destroy", strconv.Itoa(osdID)}
cmd := NewCephCommand(context, clusterName, args)
buf, err := cmd.Run()
+4 -16
View File
@@ -26,7 +26,6 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/daemon/ceph/config"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -44,18 +43,18 @@ type cephStatusChecker struct {
context *clusterd.Context
resourceName string
interval time.Duration
externalCred config.ExternalCred
cephUser string
client client.Client
namespacedName types.NamespacedName
}
// newCephStatusChecker creates a new HealthChecker object
func newCephStatusChecker(context *clusterd.Context, resourceName string, externalCred config.ExternalCred, namespacedName types.NamespacedName) *cephStatusChecker {
func newCephStatusChecker(context *clusterd.Context, resourceName string, cephUser string, namespacedName types.NamespacedName) *cephStatusChecker {
c := &cephStatusChecker{
context: context,
resourceName: resourceName,
interval: defaultStatusCheckInterval,
externalCred: externalCred,
cephUser: cephUser,
client: context.Client,
namespacedName: namespacedName,
}
@@ -95,19 +94,8 @@ func (c *cephStatusChecker) checkStatus() {
logger.Debugf("checking health of cluster")
// Set the user health check to the admin user
healthCheckUser := cephclient.AdminUsername
// This is an external cluster OR if the admin keyring is not present
// As of 1.3 an external cluster is deployed it uses a different user to check ceph's status
if c.externalCred.Username != "" && c.externalCred.Secret != "" {
if c.externalCred.Username != cephclient.AdminUsername {
healthCheckUser = c.externalCred.Username
}
}
// Check ceph's status
status, err = cephclient.StatusWithUser(c.context, c.namespacedName.Namespace, healthCheckUser)
status, err = cephclient.StatusWithUser(c.context, c.namespacedName.Namespace, c.cephUser)
if err != nil {
logger.Errorf("failed to get ceph status. %v", err)
return
+43 -17
View File
@@ -18,6 +18,7 @@ limitations under the License.
package cluster
import (
"fmt"
"path"
"strings"
"sync"
@@ -37,6 +38,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/ceph/object/bucket"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -62,6 +64,7 @@ type cluster struct {
orchestrationNeeded bool
orchMux sync.Mutex
isUpgrade bool
monitoringActivated bool
}
func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync.Mutex, ownerRef *metav1.OwnerReference) *cluster {
@@ -177,7 +180,7 @@ func (c *cluster) doOrchestration(rookImage string, cephVersion cephver.CephVers
return nil
}
func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *cephv1.CephCluster) {
func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *cephv1.CephCluster) error {
cluster.Spec = &clusterObj.Spec
// Check if the dataDirHostPath is located in the disallowed paths list
@@ -185,36 +188,62 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph
for _, b := range disallowedHostDirectories {
if cleanDataDirHostPath == b {
logger.Errorf("dataDirHostPath (given: %q) must not be used, conflicts with %q internal path", cluster.Spec.DataDirHostPath, b)
return
return nil
}
}
// The Ceph user the operator will use for management
cephUser := client.AdminUsername
// Depending on the cluster type choose the correct orchestation
if cluster.Spec.External.Enable {
err := c.configureExternalCephCluster(cluster)
if err != nil {
config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionFailure, v1.ConditionTrue, "ClusterFailure", "Failed to configure external ceph cluster")
logger.Errorf("failed to configure external ceph cluster. %v", err)
return
return errors.Wrap(err, "failed to configure external ceph cluster")
}
cephUser = cluster.Info.ExternalCred.Username
} else {
err := c.configureLocalCephCluster(cluster, clusterObj)
// If the local cluster has already been configured, immediately start monitoring the cluster.
// Test if the cluster has already been configured if the mgr deployment has been created.
// If the mgr does not exist, the mons have never been verified to be in quorum.
opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, mgr.AppName)}
mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(cluster.Namespace).List(opts)
if err == nil && len(mgrDeployments.Items) > 0 {
c.startClusterMonitoring(cluster, cephUser)
}
err = c.configureLocalCephCluster(cluster, clusterObj)
if err != nil {
logger.Errorf("failed to configure local ceph cluster. %v", err)
return
return errors.Wrap(err, "failed to configure local ceph cluster")
}
}
// Populate ClusterInfo with the last value
cluster.mons.ClusterInfo = cluster.Info
// Start the monitoring if not already started
c.startClusterMonitoring(cluster, cephUser)
return nil
}
func (c *ClusterController) startClusterMonitoring(cluster *cluster, cephUser string) {
if cluster.monitoringActivated == true {
// the cluster monitoring goroutines are already running
logger.Debugf("cluster is already being monitored for cluster %q", cluster.Namespace)
return
}
// enable the cluster monitoring goroutines once
logger.Infof("enabling cluster monitoring goroutines for cluster %q", cluster.Namespace)
cluster.monitoringActivated = true
// Start client CRD watcher
clientController := cephclient.NewClientController(c.context, cluster.Namespace)
clientController.StartWatch(cluster.stopCh)
clientToUse := client.AdminUsername
if cluster.Spec.External.Enable {
clientToUse = cluster.Info.ExternalCred.Username
}
// Start the object bucket provisioner
bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace, clientToUse)
bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace, cephUser)
// If cluster is external, pass down the user to the bucket controller
// note: the error return below is ignored and is expected to be removed from the
@@ -222,21 +251,18 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph
bucketController, _ := bucket.NewBucketController(c.context.KubeConfig, bucketProvisioner)
go bucketController.Run(cluster.stopCh)
// Populate ClusterInfo with the last value
cluster.mons.ClusterInfo = cluster.Info
// Start mon health checker
healthChecker := mon.NewHealthChecker(cluster.mons, cluster.Spec)
go healthChecker.Check(cluster.stopCh)
if !cluster.Spec.External.Enable {
// Start the osd health checker only if running OSDs in the local ceph cluster
c.osdChecker = osd.NewOSDHealthMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Info.CephVersion)
c.osdChecker = osd.NewOSDHealthMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove)
go c.osdChecker.Start(cluster.stopCh)
}
// Start the ceph status checker
cephChecker := newCephStatusChecker(c.context, cluster.Namespace, cluster.mons.ClusterInfo.ExternalCred, c.namespacedName)
cephChecker := newCephStatusChecker(c.context, cluster.Namespace, cephUser, c.namespacedName)
go cephChecker.checkCephStatus(cluster.stopCh)
}
+18 -19
View File
@@ -195,26 +195,25 @@ func populateExternalClusterInfo(context *clusterd.Context, namespace string) *c
logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
time.Sleep(externalConnectionRetry)
continue
} else {
// If an admin key was provided we don't need to load the other resources
// Some people might want to give the admin key
// The necessary users/keys/secrets will be created by Rook
// This is also done to allow backward compatibility
if isExternalHealthCheckUserAdmin(clusterInfo.AdminSecret) {
break
}
externalCred, err := mon.ValidateAndLoadExternalClusterSecrets(context, namespace)
if err != nil {
logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
logger.Debugf("%v", err)
time.Sleep(externalConnectionRetry)
continue
} else {
clusterInfo.ExternalCred = externalCred
logger.Infof("found the cluster info to connect to the external cluster. will use %q to check health and monitor status. mons=%+v", clusterInfo.ExternalCred.Username, clusterInfo.Monitors)
break
}
}
// If an admin key was provided we don't need to load the other resources
// Some people might want to give the admin key
// The necessary users/keys/secrets will be created by Rook
// This is also done to allow backward compatibility
if isExternalHealthCheckUserAdmin(clusterInfo.AdminSecret) {
clusterInfo.ExternalCred = cephconfig.ExternalCred{Username: client.AdminUsername, Secret: clusterInfo.AdminSecret}
break
}
externalCred, err := mon.ValidateAndLoadExternalClusterSecrets(context, namespace)
if err != nil {
logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
logger.Debugf("%v", err)
time.Sleep(externalConnectionRetry)
continue
}
clusterInfo.ExternalCred = externalCred
logger.Infof("found the cluster info to connect to the external cluster. will use %q to check health and monitor status. mons=%+v", clusterInfo.ExternalCred.Username, clusterInfo.Monitors)
break
}
return clusterInfo
+6 -4
View File
@@ -304,7 +304,9 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
}
// Do reconcile here!
r.clusterController.onAdd(cephCluster, ref)
if err := r.clusterController.onAdd(cephCluster, ref); err != nil {
return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile cluster %q", cephCluster.Name)
}
// Return and do not requeue
return reconcile.Result{}, nil
@@ -323,10 +325,10 @@ func NewClusterController(context *clusterd.Context, rookImage string, volumeAtt
}
}
func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.OwnerReference) {
func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.OwnerReference) error {
if hasCleanupPolicy(clusterObj) {
logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace)
return
return nil
}
cluster, ok := c.clusterMap[clusterObj.Namespace]
@@ -350,7 +352,7 @@ func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.Ow
c.csiConfigMutex.Unlock()
// Start the main ceph cluster orchestration
c.initializeCluster(cluster, clusterObj)
return c.initializeCluster(cluster, clusterObj)
}
func (c *ClusterController) requestClusterDelete(cluster *cephv1.CephCluster) (reconcile.Result, bool) {
+4 -6
View File
@@ -23,7 +23,6 @@ import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -45,12 +44,11 @@ type OSDHealthMonitor struct {
context *clusterd.Context
namespace string
removeOSDsIfOUTAndSafeToRemove bool
cephVersion cephver.CephVersion
}
// NewOSDHealthMonitor instantiates OSD monitoring
func NewOSDHealthMonitor(context *clusterd.Context, namespace string, removeOSDsIfOUTAndSafeToRemove bool, cephVersion cephver.CephVersion) *OSDHealthMonitor {
return &OSDHealthMonitor{context, namespace, removeOSDsIfOUTAndSafeToRemove, cephVersion}
func NewOSDHealthMonitor(context *clusterd.Context, namespace string, removeOSDsIfOUTAndSafeToRemove bool) *OSDHealthMonitor {
return &OSDHealthMonitor{context, namespace, removeOSDsIfOUTAndSafeToRemove}
}
// Start runs monitoring logic for osds status at set intervals
@@ -62,7 +60,7 @@ func (m *OSDHealthMonitor) Start(stopCh chan struct{}) {
logger.Debug("Checking osd processes status.")
err := m.checkOSDHealth()
if err != nil {
logger.Warningf("failed OSD status check. %v", err)
logger.Debugf("failed OSD status check. %v", err)
}
case <-stopCh:
@@ -133,7 +131,7 @@ func (m *OSDHealthMonitor) removeOSDDeploymentIfSafeToDestroy(outOSDid int) erro
return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid)
}
if len(dp.Items) != 0 {
safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.namespace, outOSDid, m.cephVersion)
safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.namespace, outOSDid)
if err != nil {
return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid)
}
+3 -12
View File
@@ -21,7 +21,6 @@ import (
"time"
"github.com/rook/rook/pkg/clusterd"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
testexec "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
@@ -88,12 +87,8 @@ func TestOSDHealthCheck(t *testing.T) {
dp, _ := context.Clientset.AppsV1().Deployments(cluster).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%v=%d", OsdIdLabelKey, 0)})
assert.Equal(t, 1, len(dp.Items))
cephVersion := cephver.CephVersion{
Major: 14,
}
// Initializing an OSD monitoring
osdMon := NewOSDHealthMonitor(context, cluster, true, cephVersion)
osdMon := NewOSDHealthMonitor(context, cluster, true)
// Run OSD monitoring routine
err := osdMon.checkOSDHealth()
@@ -107,12 +102,8 @@ func TestOSDHealthCheck(t *testing.T) {
}
func TestMonitorStart(t *testing.T) {
cephVersion := cephver.CephVersion{
Major: 14,
}
stopCh := make(chan struct{})
osdMon := NewOSDHealthMonitor(&clusterd.Context{}, "cluster", true, cephVersion)
osdMon := NewOSDHealthMonitor(&clusterd.Context{}, "cluster", true)
logger.Infof("starting osd monitor")
go osdMon.Start(stopCh)
close(stopCh)
@@ -140,7 +131,7 @@ func TestOSDRestartIfStuck(t *testing.T) {
_, err := context.Clientset.CoreV1().Pods(namespace).Create(pod)
assert.NoError(t, err)
m := NewOSDHealthMonitor(context, namespace, false, cephver.CephVersion{})
m := NewOSDHealthMonitor(context, namespace, false)
podList := &v1.PodList{Items: []v1.Pod{*pod}}
assert.NoError(t, m.restartOSDPodsIfStuck(0, podList))