From c713fd30f961953b9c954629f09cc4711c8686ab Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Wed, 29 Apr 2020 23:29:57 -0600 Subject: [PATCH] ceph: restart portable osd pod if stuck If the OSD pod is portable and is on a failed node, the node will be stuck indefinitely in the terminating state. The pod can be force deleted in this scenario in order to allow the underlying volume to be detached and then mounted on another node. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/cluster.go | 2 +- pkg/operator/ceph/cluster/controller.go | 2 +- pkg/operator/ceph/cluster/osd/health.go | 83 +++++++++++++++----- pkg/operator/ceph/cluster/osd/health_test.go | 73 ++++++++++++++++- 4 files changed, 136 insertions(+), 24 deletions(-) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 4b8a55b46..5cfbe3eec 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -224,7 +224,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph if !cluster.Spec.External.Enable { // Start the osd health checker only if running OSDs in the local ceph cluster - c.osdChecker = osd.NewMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Info.CephVersion) + c.osdChecker = osd.NewOSDHealthMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Info.CephVersion) go c.osdChecker.Start(cluster.stopCh) } diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index f26252b43..a3f1bea87 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -99,7 +99,7 @@ type ClusterController struct { addClusterCallbacks []func() error csiConfigMutex *sync.Mutex nodeStore cache.Store - osdChecker *osd.Monitor + osdChecker *osd.OSDHealthMonitor client client.Client namespacedName types.NamespacedName } diff --git a/pkg/operator/ceph/cluster/osd/health.go b/pkg/operator/ceph/cluster/osd/health.go index 5fd62a0c6..2e746bd78 100644 --- a/pkg/operator/ceph/cluster/osd/health.go +++ b/pkg/operator/ceph/cluster/osd/health.go @@ -25,7 +25,9 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" + v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -38,50 +40,49 @@ var ( healthCheckInterval = 300 * time.Second ) -// Monitor defines OSD process monitoring -type Monitor struct { +// OSDHealthMonitor defines OSD process monitoring +type OSDHealthMonitor struct { context *clusterd.Context - clusterName string + namespace string removeOSDsIfOUTAndSafeToRemove bool cephVersion cephver.CephVersion } -// NewMonitor instantiates OSD monitoring -func NewMonitor(context *clusterd.Context, clusterName string, removeOSDsIfOUTAndSafeToRemove bool, cephVersion cephver.CephVersion) *Monitor { - return &Monitor{context, clusterName, removeOSDsIfOUTAndSafeToRemove, cephVersion} +// NewOSDHealthMonitor instantiates OSD monitoring +func NewOSDHealthMonitor(context *clusterd.Context, namespace string, removeOSDsIfOUTAndSafeToRemove bool, cephVersion cephver.CephVersion) *OSDHealthMonitor { + return &OSDHealthMonitor{context, namespace, removeOSDsIfOUTAndSafeToRemove, cephVersion} } // Start runs monitoring logic for osds status at set intervals -func (m *Monitor) Start(stopCh chan struct{}) { +func (m *OSDHealthMonitor) Start(stopCh chan struct{}) { for { select { case <-time.After(healthCheckInterval): logger.Debug("Checking osd processes status.") - err := m.osdStatus() + err := m.checkOSDHealth() if err != nil { logger.Warningf("failed OSD status check. %v", err) } case <-stopCh: - logger.Infof("Stopping monitoring of OSDs in namespace %s", m.clusterName) + logger.Infof("Stopping monitoring of OSDs in namespace %s", m.namespace) return } } } // Update updates the removeOSDsIfOUTAndSafeToRemove -func (m *Monitor) Update(removeOSDsIfOUTAndSafeToRemove bool) { +func (m *OSDHealthMonitor) Update(removeOSDsIfOUTAndSafeToRemove bool) { m.removeOSDsIfOUTAndSafeToRemove = removeOSDsIfOUTAndSafeToRemove } -// OSDStatus validates osd dump output -func (m *Monitor) osdStatus() error { - osdDump, err := client.GetOSDDump(m.context, m.clusterName) +// checkOSDHealth takes action when needed if the OSDs are not healthy +func (m *OSDHealthMonitor) checkOSDHealth() error { + osdDump, err := client.GetOSDDump(m.context, m.namespace) if err != nil { return err } - logger.Debugf("osd dump %v", osdDump) for _, osdStatus := range osdDump.OSDs { id64, err := osdStatus.OSD.Int64() @@ -107,9 +108,13 @@ func (m *Monitor) osdStatus() error { if in != inStatus { logger.Debugf("osd.%d is marked 'OUT'", id) if m.removeOSDsIfOUTAndSafeToRemove { - if err := m.handleOSDMarkedOut(id); err != nil { + if err := m.removeOSDDeploymentIfSafeToDestroy(id); err != nil { logger.Errorf("error handling marked out osd osd.%d. %v", id, err) } + } else { + if err := m.restartOSDIfStuck(id); err != nil { + logger.Warningf("failed to restart OSD %d. %v", id, err) + } } } } @@ -117,9 +122,9 @@ func (m *Monitor) osdStatus() error { return nil } -func (m *Monitor) handleOSDMarkedOut(outOSDid int) error { +func (m *OSDHealthMonitor) removeOSDDeploymentIfSafeToDestroy(outOSDid int) error { label := fmt.Sprintf("ceph-osd-id=%d", outOSDid) - dp, err := k8sutil.GetDeployments(m.context.Clientset, m.clusterName, label) + dp, err := k8sutil.GetDeployments(m.context.Clientset, m.namespace, label) if err != nil { if kerrors.IsNotFound(err) { return nil @@ -127,7 +132,7 @@ func (m *Monitor) handleOSDMarkedOut(outOSDid int) error { return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid) } if len(dp.Items) != 0 { - safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.clusterName, outOSDid, m.cephVersion) + safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.namespace, outOSDid, m.cephVersion) if err != nil { return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid) } @@ -146,3 +151,45 @@ func (m *Monitor) handleOSDMarkedOut(outOSDid int) error { } return nil } + +// restartOSDIfStuck will check if a portable OSD is on a node that is not ready. +// If the pod is stuck in terminating state, go ahead and force delete the pod so K8s +// will free up the volume and allow the OSD to be restarted on another node. +func (m *OSDHealthMonitor) restartOSDIfStuck(osdID int) error { + labels := fmt.Sprintf("ceph-osd-id=%d,portable=true", osdID) + pods, err := m.context.Clientset.CoreV1().Pods(m.namespace).List(metav1.ListOptions{LabelSelector: labels}) + if err != nil { + return errors.Wrapf(err, "failed to get OSD with ID %d", osdID) + } + return m.restartOSDPodsIfStuck(osdID, pods) +} + +func (m *OSDHealthMonitor) restartOSDPodsIfStuck(osdID int, pods *v1.PodList) error { + for _, pod := range pods.Items { + logger.Debugf("checking if osd %d pod is stuck and should be force deleted", osdID) + if pod.DeletionTimestamp.IsZero() { + logger.Debugf("skipping restart of OSD %d since the pod is not deleted", osdID) + continue + } + node, err := m.context.Clientset.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) + if err != nil { + logger.Warningf("skipping restart of OSD %d since the node status is not available. %v", osdID, err) + continue + } + if k8sutil.NodeIsReady(*node) { + logger.Debugf("skipping restart of OSD %d since the node status is ready", osdID) + continue + } + + logger.Infof("force deleting pod %q that appears to be stuck terminating", pod.Name) + var gracePeriod int64 + deleteOpts := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod} + if err := m.context.Clientset.CoreV1().Pods(m.namespace).Delete(pod.Name, deleteOpts); err != nil { + logger.Warningf("pod %q deletion failed. %v", pod.Name, err) + continue + } + logger.Infof("pod %q deletion succeeded", pod.Name) + } + + return nil +} diff --git a/pkg/operator/ceph/cluster/osd/health_test.go b/pkg/operator/ceph/cluster/osd/health_test.go index 7ee797389..4de4f761d 100644 --- a/pkg/operator/ceph/cluster/osd/health_test.go +++ b/pkg/operator/ceph/cluster/osd/health_test.go @@ -18,6 +18,7 @@ package osd import ( "fmt" "testing" + "time" "github.com/rook/rook/pkg/clusterd" cephver "github.com/rook/rook/pkg/operator/ceph/version" @@ -25,12 +26,14 @@ import ( testexec "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/stretchr/testify/assert" ) -func TestOSDStatus(t *testing.T) { +func TestOSDHealthCheck(t *testing.T) { clientset := testexec.New(t, 2) cluster := "fake" @@ -90,10 +93,10 @@ func TestOSDStatus(t *testing.T) { } // Initializing an OSD monitoring - osdMon := NewMonitor(context, cluster, true, cephVersion) + osdMon := NewOSDHealthMonitor(context, cluster, true, cephVersion) // Run OSD monitoring routine - err := osdMon.osdStatus() + err := osdMon.checkOSDHealth() assert.Nil(t, err) // After creating an OSD, the dump has 1 mocked cmd and safe to destroy has 1 mocked cmd assert.Equal(t, 2, execCount) @@ -109,8 +112,70 @@ func TestMonitorStart(t *testing.T) { } stopCh := make(chan struct{}) - osdMon := NewMonitor(&clusterd.Context{}, "cluster", true, cephVersion) + osdMon := NewOSDHealthMonitor(&clusterd.Context{}, "cluster", true, cephVersion) logger.Infof("starting osd monitor") go osdMon.Start(stopCh) close(stopCh) } + +func TestOSDRestartIfStuck(t *testing.T) { + clientset := testexec.New(t, 1) + namespace := "test" + // Setting up objects needed to create OSD + context := &clusterd.Context{ + Clientset: clientset, + } + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "osd0", + Namespace: namespace, + Labels: map[string]string{ + "ceph-osd-id": "23", + "portable": "true", + }, + }, + } + pod.Spec.NodeName = "node0" + _, err := context.Clientset.CoreV1().Pods(namespace).Create(pod) + assert.NoError(t, err) + + m := NewOSDHealthMonitor(context, namespace, false, cephver.CephVersion{}) + + podList := &v1.PodList{Items: []v1.Pod{*pod}} + assert.NoError(t, m.restartOSDPodsIfStuck(0, podList)) + + // The pod should still exist since it wasn't in a deleted state + p, err := context.Clientset.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, p) + + // Add a deletion timestamp to the pod + pod.DeletionTimestamp = &metav1.Time{Time: time.Now()} + _, err = context.Clientset.CoreV1().Pods(namespace).Update(pod) + assert.NoError(t, err) + + podList = &v1.PodList{Items: []v1.Pod{*pod}} + assert.NoError(t, m.restartOSDPodsIfStuck(0, podList)) + + // The pod should still exist since the node is ready + p, err = context.Clientset.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, p) + + // Set the node to a not ready state + nodes, err := context.Clientset.CoreV1().Nodes().List(metav1.ListOptions{}) + assert.NoError(t, err) + for _, node := range nodes.Items { + node.Status.Conditions[0].Status = v1.ConditionFalse + _, err := context.Clientset.CoreV1().Nodes().Update(&node) + assert.NoError(t, err) + } + + assert.NoError(t, m.restartOSDPodsIfStuck(0, podList)) + + // The pod should be deleted since the pod is marked as deleted and the node is not ready + _, err = context.Clientset.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{}) + assert.Error(t, err) + assert.True(t, errors.IsNotFound(err)) +}