Merge pull request #5376 from travisn/restart-osd-if-stuck

Restart portable osd pod if stuck
This commit is contained in:
Travis Nielsen authored and GitHub committed 2020-05-01 08:39:03 -06:00
commit 8cdc3c103b
4 files changed
+136 -24

No files matched your search

+1 -1
View File
@@ -224,7 +224,7 @@ func (c *ClusterController) initializeCluster(cluster *cluster, clusterObj *ceph
if !cluster.Spec.External.Enable {
// Start the osd health checker only if running OSDs in the local ceph cluster
c.osdChecker = osd.NewMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Info.CephVersion)
c.osdChecker = osd.NewOSDHealthMonitor(c.context, cluster.Namespace, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Info.CephVersion)
go c.osdChecker.Start(cluster.stopCh)
}
+1 -1
View File
@@ -99,7 +99,7 @@ type ClusterController struct {
addClusterCallbacks []func() error
csiConfigMutex *sync.Mutex
nodeStore cache.Store
osdChecker *osd.Monitor
osdChecker *osd.OSDHealthMonitor
client client.Client
namespacedName types.NamespacedName
}
+65 -18
View File
@@ -25,7 +25,9 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/client"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
@@ -38,50 +40,49 @@ var (
healthCheckInterval = 300 * time.Second
)
// Monitor defines OSD process monitoring
type Monitor struct {
// OSDHealthMonitor defines OSD process monitoring
type OSDHealthMonitor struct {
context *clusterd.Context
clusterName string
namespace string
removeOSDsIfOUTAndSafeToRemove bool
cephVersion cephver.CephVersion
}
// NewMonitor instantiates OSD monitoring
func NewMonitor(context *clusterd.Context, clusterName string, removeOSDsIfOUTAndSafeToRemove bool, cephVersion cephver.CephVersion) *Monitor {
return &Monitor{context, clusterName, removeOSDsIfOUTAndSafeToRemove, cephVersion}
// NewOSDHealthMonitor instantiates OSD monitoring
func NewOSDHealthMonitor(context *clusterd.Context, namespace string, removeOSDsIfOUTAndSafeToRemove bool, cephVersion cephver.CephVersion) *OSDHealthMonitor {
return &OSDHealthMonitor{context, namespace, removeOSDsIfOUTAndSafeToRemove, cephVersion}
}
// Start runs monitoring logic for osds status at set intervals
func (m *Monitor) Start(stopCh chan struct{}) {
func (m *OSDHealthMonitor) Start(stopCh chan struct{}) {
for {
select {
case <-time.After(healthCheckInterval):
logger.Debug("Checking osd processes status.")
err := m.osdStatus()
err := m.checkOSDHealth()
if err != nil {
logger.Warningf("failed OSD status check. %v", err)
}
case <-stopCh:
logger.Infof("Stopping monitoring of OSDs in namespace %s", m.clusterName)
logger.Infof("Stopping monitoring of OSDs in namespace %s", m.namespace)
return
}
}
}
// Update updates the removeOSDsIfOUTAndSafeToRemove
func (m *Monitor) Update(removeOSDsIfOUTAndSafeToRemove bool) {
func (m *OSDHealthMonitor) Update(removeOSDsIfOUTAndSafeToRemove bool) {
m.removeOSDsIfOUTAndSafeToRemove = removeOSDsIfOUTAndSafeToRemove
}
// OSDStatus validates osd dump output
func (m *Monitor) osdStatus() error {
osdDump, err := client.GetOSDDump(m.context, m.clusterName)
// checkOSDHealth takes action when needed if the OSDs are not healthy
func (m *OSDHealthMonitor) checkOSDHealth() error {
osdDump, err := client.GetOSDDump(m.context, m.namespace)
if err != nil {
return err
}
logger.Debugf("osd dump %v", osdDump)
for _, osdStatus := range osdDump.OSDs {
id64, err := osdStatus.OSD.Int64()
@@ -107,9 +108,13 @@ func (m *Monitor) osdStatus() error {
if in != inStatus {
logger.Debugf("osd.%d is marked 'OUT'", id)
if m.removeOSDsIfOUTAndSafeToRemove {
if err := m.handleOSDMarkedOut(id); err != nil {
if err := m.removeOSDDeploymentIfSafeToDestroy(id); err != nil {
logger.Errorf("error handling marked out osd osd.%d. %v", id, err)
}
} else {
if err := m.restartOSDIfStuck(id); err != nil {
logger.Warningf("failed to restart OSD %d. %v", id, err)
}
}
}
}
@@ -117,9 +122,9 @@ func (m *Monitor) osdStatus() error {
return nil
}
func (m *Monitor) handleOSDMarkedOut(outOSDid int) error {
func (m *OSDHealthMonitor) removeOSDDeploymentIfSafeToDestroy(outOSDid int) error {
label := fmt.Sprintf("ceph-osd-id=%d", outOSDid)
dp, err := k8sutil.GetDeployments(m.context.Clientset, m.clusterName, label)
dp, err := k8sutil.GetDeployments(m.context.Clientset, m.namespace, label)
if err != nil {
if kerrors.IsNotFound(err) {
return nil
@@ -127,7 +132,7 @@ func (m *Monitor) handleOSDMarkedOut(outOSDid int) error {
return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid)
}
if len(dp.Items) != 0 {
safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.clusterName, outOSDid, m.cephVersion)
safeToDestroyOSD, err := client.OsdSafeToDestroy(m.context, m.namespace, outOSDid, m.cephVersion)
if err != nil {
return errors.Wrapf(err, "failed to get osd deployment of osd id %d", outOSDid)
}
@@ -146,3 +151,45 @@ func (m *Monitor) handleOSDMarkedOut(outOSDid int) error {
}
return nil
}
// restartOSDIfStuck will check if a portable OSD is on a node that is not ready.
// If the pod is stuck in terminating state, go ahead and force delete the pod so K8s
// will free up the volume and allow the OSD to be restarted on another node.
func (m *OSDHealthMonitor) restartOSDIfStuck(osdID int) error {
labels := fmt.Sprintf("ceph-osd-id=%d,portable=true", osdID)
pods, err := m.context.Clientset.CoreV1().Pods(m.namespace).List(metav1.ListOptions{LabelSelector: labels})
if err != nil {
return errors.Wrapf(err, "failed to get OSD with ID %d", osdID)
}
return m.restartOSDPodsIfStuck(osdID, pods)
}
func (m *OSDHealthMonitor) restartOSDPodsIfStuck(osdID int, pods *v1.PodList) error {
for _, pod := range pods.Items {
logger.Debugf("checking if osd %d pod is stuck and should be force deleted", osdID)
if pod.DeletionTimestamp.IsZero() {
logger.Debugf("skipping restart of OSD %d since the pod is not deleted", osdID)
continue
}
node, err := m.context.Clientset.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
if err != nil {
logger.Warningf("skipping restart of OSD %d since the node status is not available. %v", osdID, err)
continue
}
if k8sutil.NodeIsReady(*node) {
logger.Debugf("skipping restart of OSD %d since the node status is ready", osdID)
continue
}
logger.Infof("force deleting pod %q that appears to be stuck terminating", pod.Name)
var gracePeriod int64
deleteOpts := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}
if err := m.context.Clientset.CoreV1().Pods(m.namespace).Delete(pod.Name, deleteOpts); err != nil {
logger.Warningf("pod %q deletion failed. %v", pod.Name, err)
continue
}
logger.Infof("pod %q deletion succeeded", pod.Name)
}
return nil
}
+69 -4
View File
@@ -18,6 +18,7 @@ package osd
import (
"fmt"
"testing"
"time"
"github.com/rook/rook/pkg/clusterd"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
@@ -25,12 +26,14 @@ import (
testexec "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/stretchr/testify/assert"
)
func TestOSDStatus(t *testing.T) {
func TestOSDHealthCheck(t *testing.T) {
clientset := testexec.New(t, 2)
cluster := "fake"
@@ -90,10 +93,10 @@ func TestOSDStatus(t *testing.T) {
}
// Initializing an OSD monitoring
osdMon := NewMonitor(context, cluster, true, cephVersion)
osdMon := NewOSDHealthMonitor(context, cluster, true, cephVersion)
// Run OSD monitoring routine
err := osdMon.osdStatus()
err := osdMon.checkOSDHealth()
assert.Nil(t, err)
// After creating an OSD, the dump has 1 mocked cmd and safe to destroy has 1 mocked cmd
assert.Equal(t, 2, execCount)
@@ -109,8 +112,70 @@ func TestMonitorStart(t *testing.T) {
}
stopCh := make(chan struct{})
osdMon := NewMonitor(&clusterd.Context{}, "cluster", true, cephVersion)
osdMon := NewOSDHealthMonitor(&clusterd.Context{}, "cluster", true, cephVersion)
logger.Infof("starting osd monitor")
go osdMon.Start(stopCh)
close(stopCh)
}
func TestOSDRestartIfStuck(t *testing.T) {
clientset := testexec.New(t, 1)
namespace := "test"
// Setting up objects needed to create OSD
context := &clusterd.Context{
Clientset: clientset,
}
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "osd0",
Namespace: namespace,
Labels: map[string]string{
"ceph-osd-id": "23",
"portable": "true",
},
},
}
pod.Spec.NodeName = "node0"
_, err := context.Clientset.CoreV1().Pods(namespace).Create(pod)
assert.NoError(t, err)
m := NewOSDHealthMonitor(context, namespace, false, cephver.CephVersion{})
podList := &v1.PodList{Items: []v1.Pod{*pod}}
assert.NoError(t, m.restartOSDPodsIfStuck(0, podList))
// The pod should still exist since it wasn't in a deleted state
p, err := context.Clientset.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, p)
// Add a deletion timestamp to the pod
pod.DeletionTimestamp = &metav1.Time{Time: time.Now()}
_, err = context.Clientset.CoreV1().Pods(namespace).Update(pod)
assert.NoError(t, err)
podList = &v1.PodList{Items: []v1.Pod{*pod}}
assert.NoError(t, m.restartOSDPodsIfStuck(0, podList))
// The pod should still exist since the node is ready
p, err = context.Clientset.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, p)
// Set the node to a not ready state
nodes, err := context.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
assert.NoError(t, err)
for _, node := range nodes.Items {
node.Status.Conditions[0].Status = v1.ConditionFalse
_, err := context.Clientset.CoreV1().Nodes().Update(&node)
assert.NoError(t, err)
}
assert.NoError(t, m.restartOSDPodsIfStuck(0, podList))
// The pod should be deleted since the pod is marked as deleted and the node is not ready
_, err = context.Clientset.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
assert.Error(t, err)
assert.True(t, errors.IsNotFound(err))
}