ceph: prevent voluntary mon drain while another mon is failing over
With simultaneous node drains, a mon can go down while another mon is failing over. This results in two mons down and ceph become inaccessible. The commit makes maxUnavailable=0 while a mon is failing over and updates it back to maxUnavailable=1 after the failover. This update action is best effort. Any errors while updating the maxUnavailable mon pdb is only logged Signed-off-by: Santosh Pillai <sapillai@redhat.com>
This commit is contained in:
1 parent
42121e909e
commit
4af14f268c
2 files changed
+40
No files matched your search
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
policyv1beta1 "k8s.io/api/policy/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
)
|
||||
@@ -68,3 +69,31 @@ func (c *Cluster) createOrUpdateMonPDB(maxUnavailable int32) (controllerutil.Ope
|
||||
}
|
||||
return controllerutil.CreateOrUpdate(context.TODO(), c.context.Client, pdb, mutateFunc)
|
||||
}
|
||||
|
||||
// blockMonDrain makes MaxUnavailable in mon PDB to 0 to block any voluntary mon drains
|
||||
func (c *Cluster) blockMonDrain(request types.NamespacedName) error {
|
||||
if !c.spec.DisruptionManagement.ManagePodBudgets {
|
||||
return nil
|
||||
}
|
||||
logger.Info("prevent voluntary mon drain while failing over")
|
||||
// change MaxUnavailable mon PDB to 0
|
||||
_, err := c.createOrUpdateMonPDB(0)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to update MaxUnavailable for mon PDB %q", request.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// allowMonDrain updates the MaxUnavailable in mon PDB to 1 to allow voluntary mon drains
|
||||
func (c *Cluster) allowMonDrain(request types.NamespacedName) error {
|
||||
if !c.spec.DisruptionManagement.ManagePodBudgets {
|
||||
return nil
|
||||
}
|
||||
logger.Info("allow voluntary mon drain after failover")
|
||||
// change MaxUnavailable mon PDB to 1
|
||||
_, err := c.createOrUpdateMonPDB(1)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to update MaxUnavailable for mon PDB %q", request.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -237,10 +238,20 @@ func (c *Cluster) failMon(monCount, desiredMonCount int, name string) {
|
||||
logger.Errorf("failed to remove mon %q. %v", name, err)
|
||||
}
|
||||
} else {
|
||||
//prevent any voluntary mon drain while failing over
|
||||
if err := c.blockMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil {
|
||||
logger.Errorf("failed to block mon drain. %v", err)
|
||||
}
|
||||
|
||||
// bring up a new mon to replace the unhealthy mon
|
||||
if err := c.failoverMon(name); err != nil {
|
||||
logger.Errorf("failed to failover mon %q. %v", name, err)
|
||||
}
|
||||
|
||||
// allow any voluntary mon drain after failover
|
||||
if err := c.allowMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil {
|
||||
logger.Errorf("failed to allow mon drain. %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user