ceph: prevent voluntary mon drain while another mon is failing over

With simultaneous node drains, a mon can go down while another mon is failing over. This results in two mons down and ceph become inaccessible. The commit makes maxUnavailable=0 while a mon is failing over and updates it back to maxUnavailable=1 after the failover. This update action is best effort. Any errors while updating the maxUnavailable mon pdb is only logged

Signed-off-by: Santosh Pillai <sapillai@redhat.com>
This commit is contained in:
Santosh Pillai committed 2021-03-30 12:21:58 +05:30
1 parent 42121e909e
commit 4af14f268c
2 files changed
+40

No files matched your search

+29
View File
@@ -23,6 +23,7 @@ import (
"github.com/rook/rook/pkg/operator/k8sutil"
policyv1beta1 "k8s.io/api/policy/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)
@@ -68,3 +69,31 @@ func (c *Cluster) createOrUpdateMonPDB(maxUnavailable int32) (controllerutil.Ope
}
return controllerutil.CreateOrUpdate(context.TODO(), c.context.Client, pdb, mutateFunc)
}
// blockMonDrain makes MaxUnavailable in mon PDB to 0 to block any voluntary mon drains
func (c *Cluster) blockMonDrain(request types.NamespacedName) error {
if !c.spec.DisruptionManagement.ManagePodBudgets {
return nil
}
logger.Info("prevent voluntary mon drain while failing over")
// change MaxUnavailable mon PDB to 0
_, err := c.createOrUpdateMonPDB(0)
if err != nil {
return errors.Wrapf(err, "failed to update MaxUnavailable for mon PDB %q", request.Name)
}
return nil
}
// allowMonDrain updates the MaxUnavailable in mon PDB to 1 to allow voluntary mon drains
func (c *Cluster) allowMonDrain(request types.NamespacedName) error {
if !c.spec.DisruptionManagement.ManagePodBudgets {
return nil
}
logger.Info("allow voluntary mon drain after failover")
// change MaxUnavailable mon PDB to 1
_, err := c.createOrUpdateMonPDB(1)
if err != nil {
return errors.Wrapf(err, "failed to update MaxUnavailable for mon PDB %q", request.Name)
}
return nil
}
+11
View File
@@ -30,6 +30,7 @@ import (
"github.com/rook/rook/pkg/operator/k8sutil"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
var (
@@ -237,10 +238,20 @@ func (c *Cluster) failMon(monCount, desiredMonCount int, name string) {
logger.Errorf("failed to remove mon %q. %v", name, err)
}
} else {
//prevent any voluntary mon drain while failing over
if err := c.blockMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil {
logger.Errorf("failed to block mon drain. %v", err)
}
// bring up a new mon to replace the unhealthy mon
if err := c.failoverMon(name); err != nil {
logger.Errorf("failed to failover mon %q. %v", name, err)
}
// allow any voluntary mon drain after failover
if err := c.allowMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil {
logger.Errorf("failed to allow mon drain. %v", err)
}
}
}