diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 4b8a55b46..5cfbe3eec 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -224,7 +224,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph if !cluster.Spec.External.Enable { // Start the osd health checker only if running OSDs in the local ceph cluster - c.osdChecker = osd.NewMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Info.CephVersion) + c.osdChecker = osd.NewOSDHealthMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Info.CephVersion) go c.osdChecker.Start(cluster.stopCh) } diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index f26252b43..a3f1bea87 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -99,7 +99,7 @@ type ClusterController struct { addClusterCallbacks []func() error csiConfigMutex *sync.Mutex nodeStore cache.Store - osdChecker *osd.Monitor + osdChecker *osd.OSDHealthMonitor client client.Client namespacedName types.NamespacedName } diff --git a/pkg/operator/ceph/cluster/osd/health.go b/pkg/operator/ceph/cluster/osd/health.go index 5fd62a0c6..2e746bd78 100644 --- a/pkg/operator/ceph/cluster/osd/health.go +++ b/pkg/operator/ceph/cluster/osd/health.go @@ -25,7 +25,9 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" + v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -38,50 +40,49 @@ var ( healthCheckInterval = 300 * time.Second ) -// Monitor defines OSD process monitoring -type Monitor struct { +// OSDHealthMonitor defines OSD process monitoring +type OSDHealthMonitor struct { context *clusterd.Context - clusterName string + namespace string removeOSDsIfOUTAndSafeToRemove bool cephVersion cephver.CephVersion } -// NewMonitor instantiates OSD monitoring -func NewMonitor(context *clusterd.Context, clusterName string, removeOSDsIfOUTAndSafeToRemove bool, cephVersion cephver.CephVersion) *Monitor { - return &Monitor{context, clusterName, removeOSDsIfOUTAndSafeToRemove, cephVersion} +// NewOSDHealthMonitor instantiates OSD monitoring +func NewOSDHealthMonitor(context *clusterd.Context, namespace string, removeOSDsIfOUTAndSafeToRemove bool, cephVersion cephver.CephVersion) *OSDHealthMonitor { + return &OSDHealthMonitor{context, namespace, removeOSDsIfOUTAndSafeToRemove, cephVersion} } // Start runs monitoring logic for osds status at set intervals -func (m *Monitor) Start(stopCh chan struct{}) { +func (m *OSDHealthMonitor) Start(stopCh chan struct{}) { for { select { case <-time.After(healthCheckInterval): logger.Debug("Checking osd processes status.") - err := m.osdStatus() + err := m.checkOSDHealth() if err != nil { logger.Warningf("failed OSD status check. %v", err) } case <-stopCh: - logger.Infof("Stopping monitoring of OSDs in namespace %s", m.clusterName) + logger.Infof("Stopping monitoring of OSDs in namespace %s", m.namespace) return } } } // Update updates the removeOSDsIfOUTAndSafeToRemove -func (m *Monitor) Update(removeOSDsIfOUTAndSafeToRemove bool) { +func (m *OSDHealthMonitor) Update(removeOSDsIfOUTAndSafeToRemove bool) { m.removeOSDsIfOUTAndSafeToRemove = removeOSDsIfOUTAndSafeToRemove } -// OSDStatus validates osd dump output -func (m *Monitor) osdStatus() error { - osdDump, err := client.GetOSDDump(m.context, m.clusterName) +// checkOSDHealth takes action when needed if the OSDs are not healthy +func (m *OSDHealthMonitor) checkOSDHealth() error { + osdDump, err := client.GetOSDDump(m.context, m.namespace) if err != nil { return err } - logger.Debugf("osd dump %v", osdDump) for _, osdStatus := range osdDump.OSDs { id64, err := osdStatus.OSD.Int64() @@ -107,9 +108,13 @@ func (m *Monitor) osdStatus() error { if in != inStatus { logger.Debugf("osd.%d is marked 'OUT'", id) if m.removeOSDsIfOUTAndSafeToRemove { - if err := m.handleOSDMarkedOut(id); err != nil { + if err := m.removeOSDDeploymentIfSafeToDestroy(id); err != nil { logger.Errorf("error handling marked out osd osd.%d. %v", id, err) } + } else { + if err := m.restartOSDIfStuck(id); err != nil { + logger.Warningf("failed to restart OSD %d. %v", id, err) + } } } } @@ -117,9 +122,9 @@ func (m *Monitor) osdStatus() error { return nil } -func (m *Monitor) handleOSDMarkedOut(outOSDid int) error { +func (m *OSDHealthMonitor) removeOSDDeploymentIfSafeToDestroy(outOSDid int) error { label := fmt.Sprintf("ceph-osd-id=%d", outOSDid) - dp, err := k8sutil.GetDeployments(m.context.Clientset, m.clusterName, label) + dp, err := k8sutil.GetDeployments(m.context.Clientset, m.namespace, label) if err != nil { if kerrors.IsNotFound(err) { return nil @@ -127,7 +132,7 @@ func (m *Monitor) handleOSDMarkedOut(outOSDid int) error { return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid) } if len(dp.Items) != 0 { - safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.clusterName, outOSDid, m.cephVersion) + safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.namespace, outOSDid, m.cephVersion) if err != nil { return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid) } @@ -146,3 +151,45 @@ func (m *Monitor) handleOSDMarkedOut(outOSDid int) error { } return nil } + +// restartOSDIfStuck will check if a portable OSD is on a node that is not ready. +// If the pod is stuck in terminating state, go ahead and force delete the pod so K8s +// will free up the volume and allow the OSD to be restarted on another node. +func (m *OSDHealthMonitor) restartOSDIfStuck(osdID int) error { + labels := fmt.Sprintf("ceph-osd-id=%d,portable=true", osdID) + pods, err := m.context.Clientset.CoreV1().Pods(m.namespace).List(metav1.ListOptions{LabelSelector: labels}) + if err != nil { + return errors.Wrapf(err, "failed to get OSD with ID %d", osdID) + } + return m.restartOSDPodsIfStuck(osdID, pods) +} + +func (m *OSDHealthMonitor) restartOSDPodsIfStuck(osdID int, pods *v1.PodList) error { + for _, pod := range pods.Items { + logger.Debugf("checking if osd %d pod is stuck and should be force deleted", osdID) + if pod.DeletionTimestamp.IsZero() { + logger.Debugf("skipping restart of OSD %d since the pod is not deleted", osdID) + continue + } + node, err := m.context.Clientset.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) + if err != nil { + logger.Warningf("skipping restart of OSD %d since the node status is not available. %v", osdID, err) + continue + } + if k8sutil.NodeIsReady(*node) { + logger.Debugf("skipping restart of OSD %d since the node status is ready", osdID) + continue + } + + logger.Infof("force deleting pod %q that appears to be stuck terminating", pod.Name) + var gracePeriod int64 + deleteOpts := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod} + if err := m.context.Clientset.CoreV1().Pods(m.namespace).Delete(pod.Name, deleteOpts); err != nil { + logger.Warningf("pod %q deletion failed. %v", pod.Name, err) + continue + } + logger.Infof("pod %q deletion succeeded", pod.Name) + } + + return nil +} diff --git a/pkg/operator/ceph/cluster/osd/health_test.go b/pkg/operator/ceph/cluster/osd/health_test.go index 7ee797389..4de4f761d 100644 --- a/pkg/operator/ceph/cluster/osd/health_test.go +++ b/pkg/operator/ceph/cluster/osd/health_test.go @@ -18,6 +18,7 @@ package osd import ( "fmt" "testing" + "time" "github.com/rook/rook/pkg/clusterd" cephver "github.com/rook/rook/pkg/operator/ceph/version" @@ -25,12 +26,14 @@ import ( testexec "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/stretchr/testify/assert" ) -func TestOSDStatus(t *testing.T) { +func TestOSDHealthCheck(t *testing.T) { clientset := testexec.New(t, 2) cluster := "fake" @@ -90,10 +93,10 @@ func TestOSDStatus(t *testing.T) { } // Initializing an OSD monitoring - osdMon := NewMonitor(context, cluster, true, cephVersion) + osdMon := NewOSDHealthMonitor(context, cluster, true, cephVersion) // Run OSD monitoring routine - err := osdMon.osdStatus() + err := osdMon.checkOSDHealth() assert.Nil(t, err) // After creating an OSD, the dump has 1 mocked cmd and safe to destroy has 1 mocked cmd assert.Equal(t, 2, execCount) @@ -109,8 +112,70 @@ func TestMonitorStart(t *testing.T) { } stopCh := make(chan struct{}) - osdMon := NewMonitor(&clusterd.Context{}, "cluster", true, cephVersion) + osdMon := NewOSDHealthMonitor(&clusterd.Context{}, "cluster", true, cephVersion) logger.Infof("starting osd monitor") go osdMon.Start(stopCh) close(stopCh) } + +func TestOSDRestartIfStuck(t *testing.T) { + clientset := testexec.New(t, 1) + namespace := "test" + // Setting up objects needed to create OSD + context := &clusterd.Context{ + Clientset: clientset, + } + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "osd0", + Namespace: namespace, + Labels: map[string]string{ + "ceph-osd-id": "23", + "portable": "true", + }, + }, + } + pod.Spec.NodeName = "node0" + _, err := context.Clientset.CoreV1().Pods(namespace).Create(pod) + assert.NoError(t, err) + + m := NewOSDHealthMonitor(context, namespace, false, cephver.CephVersion{}) + + podList := &v1.PodList{Items: []v1.Pod{*pod}} + assert.NoError(t, m.restartOSDPodsIfStuck(0, podList)) + + // The pod should still exist since it wasn't in a deleted state + p, err := context.Clientset.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, p) + + // Add a deletion timestamp to the pod + pod.DeletionTimestamp = &metav1.Time{Time: time.Now()} + _, err = context.Clientset.CoreV1().Pods(namespace).Update(pod) + assert.NoError(t, err) + + podList = &v1.PodList{Items: []v1.Pod{*pod}} + assert.NoError(t, m.restartOSDPodsIfStuck(0, podList)) + + // The pod should still exist since the node is ready + p, err = context.Clientset.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, p) + + // Set the node to a not ready state + nodes, err := context.Clientset.CoreV1().Nodes().List(metav1.ListOptions{}) + assert.NoError(t, err) + for _, node := range nodes.Items { + node.Status.Conditions[0].Status = v1.ConditionFalse + _, err := context.Clientset.CoreV1().Nodes().Update(&node) + assert.NoError(t, err) + } + + assert.NoError(t, m.restartOSDPodsIfStuck(0, podList)) + + // The pod should be deleted since the pod is marked as deleted and the node is not ready + _, err = context.Clientset.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{}) + assert.Error(t, err) + assert.True(t, errors.IsNotFound(err)) +}