diff --git a/pkg/daemon/ceph/client/osd.go b/pkg/daemon/ceph/client/osd.go index 12b411c06..cfb90ead2 100644 --- a/pkg/daemon/ceph/client/osd.go +++ b/pkg/daemon/ceph/client/osd.go @@ -22,7 +22,6 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" - cephver "github.com/rook/rook/pkg/operator/ceph/version" ) type OSDUsage struct { @@ -253,7 +252,7 @@ func OSDOut(context *clusterd.Context, clusterName string, osdID int) (string, e return string(buf), err } -func OsdSafeToDestroy(context *clusterd.Context, clusterName string, osdID int, cephVersion cephver.CephVersion) (bool, error) { +func OsdSafeToDestroy(context *clusterd.Context, clusterName string, osdID int) (bool, error) { args := []string{"osd", "safe-to-destroy", strconv.Itoa(osdID)} cmd := NewCephCommand(context, clusterName, args) buf, err := cmd.Run() diff --git a/pkg/operator/ceph/cluster/cephstatus.go b/pkg/operator/ceph/cluster/cephstatus.go index a08ef82c5..68b7fbdae 100644 --- a/pkg/operator/ceph/cluster/cephstatus.go +++ b/pkg/operator/ceph/cluster/cephstatus.go @@ -26,7 +26,6 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" - "github.com/rook/rook/pkg/daemon/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" cephver "github.com/rook/rook/pkg/operator/ceph/version" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -44,18 +43,18 @@ type cephStatusChecker struct { context *clusterd.Context resourceName string interval time.Duration - externalCred config.ExternalCred + cephUser string client client.Client namespacedName types.NamespacedName } // newCephStatusChecker creates a new HealthChecker object -func newCephStatusChecker(context *clusterd.Context, resourceName string, externalCred config.ExternalCred, namespacedName types.NamespacedName) *cephStatusChecker { +func newCephStatusChecker(context *clusterd.Context, resourceName string, cephUser string, namespacedName types.NamespacedName) *cephStatusChecker { c := &cephStatusChecker{ context: context, resourceName: resourceName, interval: defaultStatusCheckInterval, - externalCred: externalCred, + cephUser: cephUser, client: context.Client, namespacedName: namespacedName, } @@ -95,19 +94,8 @@ func (c *cephStatusChecker) checkStatus() { logger.Debugf("checking health of cluster") - // Set the user health check to the admin user - healthCheckUser := cephclient.AdminUsername - - // This is an external cluster OR if the admin keyring is not present - // As of 1.3 an external cluster is deployed it uses a different user to check ceph's status - if c.externalCred.Username != "" && c.externalCred.Secret != "" { - if c.externalCred.Username != cephclient.AdminUsername { - healthCheckUser = c.externalCred.Username - } - } - // Check ceph's status - status, err = cephclient.StatusWithUser(c.context, c.namespacedName.Namespace, healthCheckUser) + status, err = cephclient.StatusWithUser(c.context, c.namespacedName.Namespace, c.cephUser) if err != nil { logger.Errorf("failed to get ceph status. %v", err) return diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 2f8814a38..ed4ae4fe8 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -18,6 +18,7 @@ limitations under the License. package cluster import ( + "fmt" "path" "strings" "sync" @@ -37,6 +38,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/csi" "github.com/rook/rook/pkg/operator/ceph/object/bucket" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,6 +64,7 @@ type cluster struct { orchestrationNeeded bool orchMux sync.Mutex isUpgrade bool + monitoringActivated bool } func newCluster(c *cephv1.CephCluster, context *clusterd.Context, csiMutex *sync.Mutex, ownerRef *metav1.OwnerReference) *cluster { @@ -177,7 +180,7 @@ func (c *cluster) doOrchestration(rookImage string, cephVersion cephver.CephVers return nil } -func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *cephv1.CephCluster) { +func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *cephv1.CephCluster) error { cluster.Spec = &clusterObj.Spec // Check if the dataDirHostPath is located in the disallowed paths list @@ -185,36 +188,62 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph for _, b := range disallowedHostDirectories { if cleanDataDirHostPath == b { logger.Errorf("dataDirHostPath (given: %q) must not be used, conflicts with %q internal path", cluster.Spec.DataDirHostPath, b) - return + return nil } } + // The Ceph user the operator will use for management + cephUser := client.AdminUsername + // Depending on the cluster type choose the correct orchestation if cluster.Spec.External.Enable { err := c.configureExternalCephCluster(cluster) if err != nil { config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionFailure, v1.ConditionTrue, "ClusterFailure", "Failed to configure external ceph cluster") - logger.Errorf("failed to configure external ceph cluster. %v", err) - return + return errors.Wrap(err, "failed to configure external ceph cluster") } + cephUser = cluster.Info.ExternalCred.Username } else { - err := c.configureLocalCephCluster(cluster, clusterObj) + // If the local cluster has already been configured, immediately start monitoring the cluster. + // Test if the cluster has already been configured if the mgr deployment has been created. + // If the mgr does not exist, the mons have never been verified to be in quorum. + opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, mgr.AppName)} + mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(cluster.Namespace).List(opts) + if err == nil && len(mgrDeployments.Items) > 0 { + c.startClusterMonitoring(cluster, cephUser) + } + + err = c.configureLocalCephCluster(cluster, clusterObj) if err != nil { - logger.Errorf("failed to configure local ceph cluster. %v", err) - return + return errors.Wrap(err, "failed to configure local ceph cluster") } } + // Populate ClusterInfo with the last value + cluster.mons.ClusterInfo = cluster.Info + + // Start the monitoring if not already started + c.startClusterMonitoring(cluster, cephUser) + return nil +} + +func (c *ClusterController) startClusterMonitoring(cluster *cluster, cephUser string) { + if cluster.monitoringActivated == true { + // the cluster monitoring goroutines are already running + logger.Debugf("cluster is already being monitored for cluster %q", cluster.Namespace) + return + } + + // enable the cluster monitoring goroutines once + logger.Infof("enabling cluster monitoring goroutines for cluster %q", cluster.Namespace) + cluster.monitoringActivated = true + // Start client CRD watcher clientController := cephclient.NewClientController(c.context, cluster.Namespace) clientController.StartWatch(cluster.stopCh) - clientToUse := client.AdminUsername - if cluster.Spec.External.Enable { - clientToUse = cluster.Info.ExternalCred.Username - } // Start the object bucket provisioner - bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace, clientToUse) + bucketProvisioner := bucket.NewProvisioner(c.context, cluster.Namespace, cephUser) // If cluster is external, pass down the user to the bucket controller // note: the error return below is ignored and is expected to be removed from the @@ -222,21 +251,18 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph bucketController, _ := bucket.NewBucketController(c.context.KubeConfig, bucketProvisioner) go bucketController.Run(cluster.stopCh) - // Populate ClusterInfo with the last value - cluster.mons.ClusterInfo = cluster.Info - // Start mon health checker healthChecker := mon.NewHealthChecker(cluster.mons, cluster.Spec) go healthChecker.Check(cluster.stopCh) if !cluster.Spec.External.Enable { // Start the osd health checker only if running OSDs in the local ceph cluster - c.osdChecker = osd.NewOSDHealthMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Info.CephVersion) + c.osdChecker = osd.NewOSDHealthMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove) go c.osdChecker.Start(cluster.stopCh) } // Start the ceph status checker - cephChecker := newCephStatusChecker(c.context, cluster.Namespace, cluster.mons.ClusterInfo.ExternalCred, c.namespacedName) + cephChecker := newCephStatusChecker(c.context, cluster.Namespace, cephUser, c.namespacedName) go cephChecker.checkCephStatus(cluster.stopCh) } diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go index 046b4a086..9721d30f0 100644 --- a/pkg/operator/ceph/cluster/cluster_external.go +++ b/pkg/operator/ceph/cluster/cluster_external.go @@ -195,26 +195,25 @@ func populateExternalClusterInfo(context *clusterd.Context, namespace string) *c logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String()) time.Sleep(externalConnectionRetry) continue - } else { - // If an admin key was provided we don't need to load the other resources - // Some people might want to give the admin key - // The necessary users/keys/secrets will be created by Rook - // This is also done to allow backward compatibility - if isExternalHealthCheckUserAdmin(clusterInfo.AdminSecret) { - break - } - externalCred, err := mon.ValidateAndLoadExternalClusterSecrets(context, namespace) - if err != nil { - logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String()) - logger.Debugf("%v", err) - time.Sleep(externalConnectionRetry) - continue - } else { - clusterInfo.ExternalCred = externalCred - logger.Infof("found the cluster info to connect to the external cluster. will use %q to check health and monitor status. mons=%+v", clusterInfo.ExternalCred.Username, clusterInfo.Monitors) - break - } } + // If an admin key was provided we don't need to load the other resources + // Some people might want to give the admin key + // The necessary users/keys/secrets will be created by Rook + // This is also done to allow backward compatibility + if isExternalHealthCheckUserAdmin(clusterInfo.AdminSecret) { + clusterInfo.ExternalCred = cephconfig.ExternalCred{Username: client.AdminUsername, Secret: clusterInfo.AdminSecret} + break + } + externalCred, err := mon.ValidateAndLoadExternalClusterSecrets(context, namespace) + if err != nil { + logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String()) + logger.Debugf("%v", err) + time.Sleep(externalConnectionRetry) + continue + } + clusterInfo.ExternalCred = externalCred + logger.Infof("found the cluster info to connect to the external cluster. will use %q to check health and monitor status. mons=%+v", clusterInfo.ExternalCred.Username, clusterInfo.Monitors) + break } return clusterInfo diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index b1225a81b..b5ab3b10d 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -304,7 +304,9 @@ func (r *ReconcileCephCluster) reconcile(request reconcile.Request) (reconcile.R } // Do reconcile here! - r.clusterController.onAdd(cephCluster, ref) + if err := r.clusterController.onAdd(cephCluster, ref); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile cluster %q", cephCluster.Name) + } // Return and do not requeue return reconcile.Result{}, nil @@ -323,10 +325,10 @@ func NewClusterController(context *clusterd.Context, rookImage string, volumeAtt } } -func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.OwnerReference) { +func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.OwnerReference) error { if hasCleanupPolicy(clusterObj) { logger.Infof("skipping orchestration for cluster object %q in namespace %q because its cleanup policy is set", clusterObj.Name, clusterObj.Namespace) - return + return nil } cluster, ok := c.clusterMap[clusterObj.Namespace] @@ -350,7 +352,7 @@ func (c *ClusterController) onAdd(clusterObj *cephv1.CephCluster, ref *metav1.Ow c.csiConfigMutex.Unlock() // Start the main ceph cluster orchestration - c.initializeCluster(cluster, clusterObj) + return c.initializeCluster(cluster, clusterObj) } func (c *ClusterController) requestClusterDelete(cluster *cephv1.CephCluster) (reconcile.Result, bool) { diff --git a/pkg/operator/ceph/cluster/osd/health.go b/pkg/operator/ceph/cluster/osd/health.go index 83ac953d7..ca3cad6aa 100644 --- a/pkg/operator/ceph/cluster/osd/health.go +++ b/pkg/operator/ceph/cluster/osd/health.go @@ -23,7 +23,6 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" - cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -45,12 +44,11 @@ type OSDHealthMonitor struct { context *clusterd.Context namespace string removeOSDsIfOUTAndSafeToRemove bool - cephVersion cephver.CephVersion } // NewOSDHealthMonitor instantiates OSD monitoring -func NewOSDHealthMonitor(context *clusterd.Context, namespace string, removeOSDsIfOUTAndSafeToRemove bool, cephVersion cephver.CephVersion) *OSDHealthMonitor { - return &OSDHealthMonitor{context, namespace, removeOSDsIfOUTAndSafeToRemove, cephVersion} +func NewOSDHealthMonitor(context *clusterd.Context, namespace string, removeOSDsIfOUTAndSafeToRemove bool) *OSDHealthMonitor { + return &OSDHealthMonitor{context, namespace, removeOSDsIfOUTAndSafeToRemove} } // Start runs monitoring logic for osds status at set intervals @@ -62,7 +60,7 @@ func (m *OSDHealthMonitor) Start(stopCh chan struct{}) { logger.Debug("Checking osd processes status.") err := m.checkOSDHealth() if err != nil { - logger.Warningf("failed OSD status check. %v", err) + logger.Debugf("failed OSD status check. %v", err) } case <-stopCh: @@ -133,7 +131,7 @@ func (m *OSDHealthMonitor) removeOSDDeploymentIfSafeToDestroy(outOSDid int) erro return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid) } if len(dp.Items) != 0 { - safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.namespace, outOSDid, m.cephVersion) + safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.namespace, outOSDid) if err != nil { return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid) } diff --git a/pkg/operator/ceph/cluster/osd/health_test.go b/pkg/operator/ceph/cluster/osd/health_test.go index 742bc0f85..fd159dd48 100644 --- a/pkg/operator/ceph/cluster/osd/health_test.go +++ b/pkg/operator/ceph/cluster/osd/health_test.go @@ -21,7 +21,6 @@ import ( "time" "github.com/rook/rook/pkg/clusterd" - cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" testexec "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -88,12 +87,8 @@ func TestOSDHealthCheck(t *testing.T) { dp, _ := context.Clientset.AppsV1().Deployments(cluster).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%v=%d", OsdIdLabelKey, 0)}) assert.Equal(t, 1, len(dp.Items)) - cephVersion := cephver.CephVersion{ - Major: 14, - } - // Initializing an OSD monitoring - osdMon := NewOSDHealthMonitor(context, cluster, true, cephVersion) + osdMon := NewOSDHealthMonitor(context, cluster, true) // Run OSD monitoring routine err := osdMon.checkOSDHealth() @@ -107,12 +102,8 @@ func TestOSDHealthCheck(t *testing.T) { } func TestMonitorStart(t *testing.T) { - cephVersion := cephver.CephVersion{ - Major: 14, - } - stopCh := make(chan struct{}) - osdMon := NewOSDHealthMonitor(&clusterd.Context{}, "cluster", true, cephVersion) + osdMon := NewOSDHealthMonitor(&clusterd.Context{}, "cluster", true) logger.Infof("starting osd monitor") go osdMon.Start(stopCh) close(stopCh) @@ -140,7 +131,7 @@ func TestOSDRestartIfStuck(t *testing.T) { _, err := context.Clientset.CoreV1().Pods(namespace).Create(pod) assert.NoError(t, err) - m := NewOSDHealthMonitor(context, namespace, false, cephver.CephVersion{}) + m := NewOSDHealthMonitor(context, namespace, false) podList := &v1.PodList{Items: []v1.Pod{*pod}} assert.NoError(t, m.restartOSDPodsIfStuck(0, podList))