Merge pull request #5490 from travisn/monitoring-goroutines
Only run the cluster monitoring goroutines once
This commit is contained in:
7 files changed
+79
-76
No files matched your search
@@ -22,7 +22,6 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
)
|
||||
|
||||
type OSDUsage struct {
|
||||
@@ -253,7 +252,7 @@ func OSDOut(context *clusterd.Context, clusterName string, osdID int) (string, e
|
||||
return string(buf), err
|
||||
}
|
||||
|
||||
func OsdSafeToDestroy(context *clusterd.Context, clusterName string, osdID int, cephVersion cephver.CephVersion) (bool, error) {
|
||||
func OsdSafeToDestroy(context *clusterd.Context, clusterName string, osdID int) (bool, error) {
|
||||
args := []string{"osd", "safe-to-destroy", strconv.Itoa(osdID)}
|
||||
cmd := NewCephCommand(context, clusterName, args)
|
||||
buf, err := cmd.Run()
|
||||
|
||||
@@ -26,7 +26,6 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -44,18 +43,18 @@ type cephStatusChecker struct {
|
||||
context *clusterd.Context
|
||||
resourceName string
|
||||
interval time.Duration
|
||||
externalCred config.ExternalCred
|
||||
cephUser string
|
||||
client client.Client
|
||||
namespacedName types.NamespacedName
|
||||
}
|
||||
|
||||
// newCephStatusChecker creates a new HealthChecker object
|
||||
func newCephStatusChecker(context *clusterd.Context, resourceName string, externalCred config.ExternalCred, namespacedName types.NamespacedName) *cephStatusChecker {
|
||||
func newCephStatusChecker(context *clusterd.Context, resourceName string, cephUser string, namespacedName types.NamespacedName) *cephStatusChecker {
|
||||
c := &cephStatusChecker{
|
||||
context: context,
|
||||
resourceName: resourceName,
|
||||
interval: defaultStatusCheckInterval,
|
||||
externalCred: externalCred,
|
||||
cephUser: cephUser,
|
||||
client: context.Client,
|
||||
namespacedName: namespacedName,
|
||||
}
|
||||
@@ -95,19 +94,8 @@ func (c *cephStatusChecker) checkStatus() {
|
||||
|
||||
logger.Debugf("checking health of cluster")
|
||||
|
||||
// Set the user health check to the admin user
|
||||
healthCheckUser := cephclient.AdminUsername
|
||||
|
||||
// This is an external cluster OR if the admin keyring is not present
|
||||
// As of 1.3 an external cluster is deployed it uses a different user to check ceph's status
|
||||
if c.externalCred.Username != "" && c.externalCred.Secret != "" {
|
||||
if c.externalCred.Username != cephclient.AdminUsername {
|
||||
healthCheckUser = c.externalCred.Username
|
||||
}
|
||||
}
|
||||
|
||||
// Check ceph's status
|
||||
status, err = cephclient.StatusWithUser(c.context, c.namespacedName.Namespace, healthCheckUser)
|
||||
status, err = cephclient.StatusWithUser(c.context, c.namespacedName.Namespace, c.cephUser)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get ceph status. %v", err)
|
||||
return
|
||||
|
||||
@@ -18,6 +18,7 @@ limitations under the License.
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -37,6 +38,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object/bucket"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -62,6 +64,7 @@ type cluster struct {
|
||||
orchestrationNeeded bool
|
||||
orchMux sync.Mutex
|
||||
isUpgrade bool
|
||||
monitoringActivated bool
|
||||
}
|
||||
|
||||
func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync.Mutex, ownerRef *metav1.OwnerReference) *cluster {
|
||||
@@ -177,7 +180,7 @@ func (c *cluster) doOrchestration(rookImage string, cephVersion cephver.CephVers
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *cephv1.CephCluster) {
|
||||
func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *cephv1.CephCluster) error {
|
||||
cluster.Spec = &clusterObj.Spec
|
||||
|
||||
// Check if the dataDirHostPath is located in the disallowed paths list
|
||||
@@ -185,36 +188,62 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph
|
||||
for _, b := range disallowedHostDirectories {
|
||||
if cleanDataDirHostPath == b {
|
||||
logger.Errorf("dataDirHostPath (given: %q) must not be used, conflicts with %q internal path", cluster.Spec.DataDirHostPath, b)
|
||||
return
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// The Ceph user the operator will use for management
|
||||
cephUser := client.AdminUsername
|
||||
|
||||
// Depending on the cluster type choose the correct orchestation
|
||||
if cluster.Spec.External.Enable {
|
||||
err := c.configureExternalCephCluster(cluster)
|
||||
if err != nil {
|
||||
config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionFailure, v1.ConditionTrue, "ClusterFailure", "Failed to configure external ceph cluster")
|
||||
logger.Errorf("failed to configure external ceph cluster. %v", err)
|
||||
return
|
||||
return errors.Wrap(err, "failed to configure external ceph cluster")
|
||||
}
|
||||
cephUser = cluster.Info.ExternalCred.Username
|
||||
} else {
|
||||
err := c.configureLocalCephCluster(cluster, clusterObj)
|
||||
// If the local cluster has already been configured, immediately start monitoring the cluster.
|
||||
// Test if the cluster has already been configured if the mgr deployment has been created.
|
||||
// If the mgr does not exist, the mons have never been verified to be in quorum.
|
||||
opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, mgr.AppName)}
|
||||
mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(cluster.Namespace).List(opts)
|
||||
if err == nil && len(mgrDeployments.Items) > 0 {
|
||||
c.startClusterMonitoring(cluster, cephUser)
|
||||
}
|
||||
|
||||
err = c.configureLocalCephCluster(cluster, clusterObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to configure local ceph cluster. %v", err)
|
||||
return
|
||||
return errors.Wrap(err, "failed to configure local ceph cluster")
|
||||
}
|
||||
}
|
||||
|
||||
// Populate ClusterInfo with the last value
|
||||
cluster.mons.ClusterInfo = cluster.Info
|
||||
|
||||
// Start the monitoring if not already started
|
||||
c.startClusterMonitoring(cluster, cephUser)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ClusterController) startClusterMonitoring(cluster *cluster, cephUser string) {
|
||||
if cluster.monitoringActivated == true {
|
||||
// the cluster monitoring goroutines are already running
|
||||
logger.Debugf("cluster is already being monitored for cluster %q", cluster.Namespace)
|
||||
return
|
||||
}
|
||||
|
||||
// enable the cluster monitoring goroutines once
|
||||
logger.Infof("enabling cluster monitoring goroutines for cluster %q", cluster.Namespace)
|
||||
cluster.monitoringActivated = true
|
||||
|
||||
// Start client CRD watcher
|
||||
clientController := cephclient.NewClientController(c.context, cluster.Namespace)
|
||||
clientController.StartWatch(cluster.stopCh)
|
||||
|
||||
clientToUse := client.AdminUsername
|
||||
if cluster.Spec.External.Enable {
|
||||
clientToUse = cluster.Info.ExternalCred.Username
|
||||
}
|
||||
// Start the object bucket provisioner
|
||||
bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace, clientToUse)
|
||||
bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace, cephUser)
|
||||
// If cluster is external, pass down the user to the bucket controller
|
||||
|
||||
// note: the error return below is ignored and is expected to be removed from the
|
||||
@@ -222,21 +251,18 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph
|
||||
bucketController, _ := bucket.NewBucketController(c.context.KubeConfig, bucketProvisioner)
|
||||
go bucketController.Run(cluster.stopCh)
|
||||
|
||||
// Populate ClusterInfo with the last value
|
||||
cluster.mons.ClusterInfo = cluster.Info
|
||||
|
||||
// Start mon health checker
|
||||
healthChecker := mon.NewHealthChecker(cluster.mons, cluster.Spec)
|
||||
go healthChecker.Check(cluster.stopCh)
|
||||
|
||||
if !cluster.Spec.External.Enable {
|
||||
// Start the osd health checker only if running OSDs in the local ceph cluster
|
||||
c.osdChecker = osd.NewOSDHealthMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Info.CephVersion)
|
||||
c.osdChecker = osd.NewOSDHealthMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove)
|
||||
go c.osdChecker.Start(cluster.stopCh)
|
||||
}
|
||||
|
||||
// Start the ceph status checker
|
||||
cephChecker := newCephStatusChecker(c.context, cluster.Namespace, cluster.mons.ClusterInfo.ExternalCred, c.namespacedName)
|
||||
cephChecker := newCephStatusChecker(c.context, cluster.Namespace, cephUser, c.namespacedName)
|
||||
go cephChecker.checkCephStatus(cluster.stopCh)
|
||||
}
|
||||
|
||||
|
||||
@@ -195,26 +195,25 @@ func populateExternalClusterInfo(context *clusterd.Context, namespace string) *c
|
||||
logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
|
||||
time.Sleep(externalConnectionRetry)
|
||||
continue
|
||||
} else {
|
||||
// If an admin key was provided we don't need to load the other resources
|
||||
// Some people might want to give the admin key
|
||||
// The necessary users/keys/secrets will be created by Rook
|
||||
// This is also done to allow backward compatibility
|
||||
if isExternalHealthCheckUserAdmin(clusterInfo.AdminSecret) {
|
||||
break
|
||||
}
|
||||
externalCred, err := mon.ValidateAndLoadExternalClusterSecrets(context, namespace)
|
||||
if err != nil {
|
||||
logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
|
||||
logger.Debugf("%v", err)
|
||||
time.Sleep(externalConnectionRetry)
|
||||
continue
|
||||
} else {
|
||||
clusterInfo.ExternalCred = externalCred
|
||||
logger.Infof("found the cluster info to connect to the external cluster. will use %q to check health and monitor status. mons=%+v", clusterInfo.ExternalCred.Username, clusterInfo.Monitors)
|
||||
break
|
||||
}
|
||||
}
|
||||
// If an admin key was provided we don't need to load the other resources
|
||||
// Some people might want to give the admin key
|
||||
// The necessary users/keys/secrets will be created by Rook
|
||||
// This is also done to allow backward compatibility
|
||||
if isExternalHealthCheckUserAdmin(clusterInfo.AdminSecret) {
|
||||
clusterInfo.ExternalCred = cephconfig.ExternalCred{Username: client.AdminUsername, Secret: clusterInfo.AdminSecret}
|
||||
break
|
||||
}
|
||||
externalCred, err := mon.ValidateAndLoadExternalClusterSecrets(context, namespace)
|
||||
if err != nil {
|
||||
logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
|
||||
logger.Debugf("%v", err)
|
||||
time.Sleep(externalConnectionRetry)
|
||||
continue
|
||||
}
|
||||
clusterInfo.ExternalCred = externalCred
|
||||
logger.Infof("found the cluster info to connect to the external cluster. will use %q to check health and monitor status. mons=%+v", clusterInfo.ExternalCred.Username, clusterInfo.Monitors)
|
||||
break
|
||||
}
|
||||
|
||||
return clusterInfo
|
||||
|
||||
@@ -304,7 +304,9 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R
|
||||
}
|
||||
|
||||
// Do reconcile here!
|
||||
r.clusterController.onAdd(cephCluster, ref)
|
||||
if err := r.clusterController.onAdd(cephCluster, ref); err != nil {
|
||||
return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile cluster %q", cephCluster.Name)
|
||||
}
|
||||
|
||||
// Return and do not requeue
|
||||
return reconcile.Result{}, nil
|
||||
@@ -323,10 +325,10 @@ func NewClusterController(context *clusterd.Context, rookImage string, volumeAtt
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.OwnerReference) {
|
||||
func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.OwnerReference) error {
|
||||
if hasCleanupPolicy(clusterObj) {
|
||||
logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace)
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
cluster, ok := c.clusterMap[clusterObj.Namespace]
|
||||
@@ -350,7 +352,7 @@ func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.Ow
|
||||
c.csiConfigMutex.Unlock()
|
||||
|
||||
// Start the main ceph cluster orchestration
|
||||
c.initializeCluster(cluster, clusterObj)
|
||||
return c.initializeCluster(cluster, clusterObj)
|
||||
}
|
||||
|
||||
func (c *ClusterController) requestClusterDelete(cluster *cephv1.CephCluster) (reconcile.Result, bool) {
|
||||
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -45,12 +44,11 @@ type OSDHealthMonitor struct {
|
||||
context *clusterd.Context
|
||||
namespace string
|
||||
removeOSDsIfOUTAndSafeToRemove bool
|
||||
cephVersion cephver.CephVersion
|
||||
}
|
||||
|
||||
// NewOSDHealthMonitor instantiates OSD monitoring
|
||||
func NewOSDHealthMonitor(context *clusterd.Context, namespace string, removeOSDsIfOUTAndSafeToRemove bool, cephVersion cephver.CephVersion) *OSDHealthMonitor {
|
||||
return &OSDHealthMonitor{context, namespace, removeOSDsIfOUTAndSafeToRemove, cephVersion}
|
||||
func NewOSDHealthMonitor(context *clusterd.Context, namespace string, removeOSDsIfOUTAndSafeToRemove bool) *OSDHealthMonitor {
|
||||
return &OSDHealthMonitor{context, namespace, removeOSDsIfOUTAndSafeToRemove}
|
||||
}
|
||||
|
||||
// Start runs monitoring logic for osds status at set intervals
|
||||
@@ -62,7 +60,7 @@ func (m *OSDHealthMonitor) Start(stopCh chan struct{}) {
|
||||
logger.Debug("Checking osd processes status.")
|
||||
err := m.checkOSDHealth()
|
||||
if err != nil {
|
||||
logger.Warningf("failed OSD status check. %v", err)
|
||||
logger.Debugf("failed OSD status check. %v", err)
|
||||
}
|
||||
|
||||
case <-stopCh:
|
||||
@@ -133,7 +131,7 @@ func (m *OSDHealthMonitor) removeOSDDeploymentIfSafeToDestroy(outOSDid int) erro
|
||||
return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid)
|
||||
}
|
||||
if len(dp.Items) != 0 {
|
||||
safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.namespace, outOSDid, m.cephVersion)
|
||||
safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.namespace, outOSDid)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid)
|
||||
}
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
testexec "github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
@@ -88,12 +87,8 @@ func TestOSDHealthCheck(t *testing.T) {
|
||||
dp, _ := context.Clientset.AppsV1().Deployments(cluster).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%v=%d", OsdIdLabelKey, 0)})
|
||||
assert.Equal(t, 1, len(dp.Items))
|
||||
|
||||
cephVersion := cephver.CephVersion{
|
||||
Major: 14,
|
||||
}
|
||||
|
||||
// Initializing an OSD monitoring
|
||||
osdMon := NewOSDHealthMonitor(context, cluster, true, cephVersion)
|
||||
osdMon := NewOSDHealthMonitor(context, cluster, true)
|
||||
|
||||
// Run OSD monitoring routine
|
||||
err := osdMon.checkOSDHealth()
|
||||
@@ -107,12 +102,8 @@ func TestOSDHealthCheck(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMonitorStart(t *testing.T) {
|
||||
cephVersion := cephver.CephVersion{
|
||||
Major: 14,
|
||||
}
|
||||
|
||||
stopCh := make(chan struct{})
|
||||
osdMon := NewOSDHealthMonitor(&clusterd.Context{}, "cluster", true, cephVersion)
|
||||
osdMon := NewOSDHealthMonitor(&clusterd.Context{}, "cluster", true)
|
||||
logger.Infof("starting osd monitor")
|
||||
go osdMon.Start(stopCh)
|
||||
close(stopCh)
|
||||
@@ -140,7 +131,7 @@ func TestOSDRestartIfStuck(t *testing.T) {
|
||||
_, err := context.Clientset.CoreV1().Pods(namespace).Create(pod)
|
||||
assert.NoError(t, err)
|
||||
|
||||
m := NewOSDHealthMonitor(context, namespace, false, cephver.CephVersion{})
|
||||
m := NewOSDHealthMonitor(context, namespace, false)
|
||||
|
||||
podList := &v1.PodList{Items: []v1.Pod{*pod}}
|
||||
assert.NoError(t, m.restartOSDPodsIfStuck(0, podList))
|
||||
|
||||
Reference in new issue
Block a user