From 3c6bae56b70df9ef440936dde253f379f9ae2ea5 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Wed, 6 May 2020 13:35:17 -0600 Subject: [PATCH] ceph: delete terminating osd pods even if not out When an OSD pod is stuck in terminating state, there is no need to check if the OSD is marked as out before force deleting. As long as the osd is down we will force delete the pod. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/osd/health.go | 15 ++++++++------- pkg/operator/ceph/cluster/osd/health_test.go | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/operator/ceph/cluster/osd/health.go b/pkg/operator/ceph/cluster/osd/health.go index 2e746bd78..83ac953d7 100644 --- a/pkg/operator/ceph/cluster/osd/health.go +++ b/pkg/operator/ceph/cluster/osd/health.go @@ -98,11 +98,16 @@ func (m *OSDHealthMonitor) checkOSDHealth() error { return err } - if status != upStatus { - logger.Debugf("osd.%d is marked 'DOWN'", id) - } else { + if status == upStatus { logger.Debugf("osd.%d is healthy.", id) + continue + } + logger.Debugf("osd.%d is marked 'DOWN'", id) + + // check if the down osd is stuck terminating + if err := m.restartOSDIfStuck(id); err != nil { + logger.Warningf("failed to restart OSD %d. %v", id, err) } if in != inStatus { @@ -111,10 +116,6 @@ func (m *OSDHealthMonitor) checkOSDHealth() error { if err := m.removeOSDDeploymentIfSafeToDestroy(id); err != nil { logger.Errorf("error handling marked out osd osd.%d. %v", id, err) } - } else { - if err := m.restartOSDIfStuck(id); err != nil { - logger.Warningf("failed to restart OSD %d. %v", id, err) - } } } } diff --git a/pkg/operator/ceph/cluster/osd/health_test.go b/pkg/operator/ceph/cluster/osd/health_test.go index 4de4f761d..742bc0f85 100644 --- a/pkg/operator/ceph/cluster/osd/health_test.go +++ b/pkg/operator/ceph/cluster/osd/health_test.go @@ -48,7 +48,7 @@ func TestOSDHealthCheck(t *testing.T) { execCount++ if args[1] == "dump" { // Mock executor for OSD Dump command, returning an osd in Down state - return `{"OSDs": [{"OSD": 0, "Up": 1, "In": 0}]}`, nil + return `{"OSDs": [{"OSD": 0, "Up": 0, "In": 0}]}`, nil } else if args[1] == "safe-to-destroy" { // Mock executor for OSD Dump command, returning an osd in Down state return `{"safe_to_destroy":[0],"active":[],"missing_stats":[],"stored_pgs":[]}`, nil