From 4af14f268c28337f83ee3c4effdc5daf87cc263e Mon Sep 17 00:00:00 2001 From: Santosh Pillai Date: Tue, 30 Mar 2021 10:16:50 +0530 Subject: [PATCH] ceph: prevent voluntary mon drain while another mon is failing over With simultaneous node drains, a mon can go down while another mon is failing over. This results in two mons down and ceph become inaccessible. The commit makes maxUnavailable=0 while a mon is failing over and updates it back to maxUnavailable=1 after the failover. This update action is best effort. Any errors while updating the maxUnavailable mon pdb is only logged Signed-off-by: Santosh Pillai --- pkg/operator/ceph/cluster/mon/drain.go | 29 +++++++++++++++++++++++++ pkg/operator/ceph/cluster/mon/health.go | 11 ++++++++++ 2 files changed, 40 insertions(+) diff --git a/pkg/operator/ceph/cluster/mon/drain.go b/pkg/operator/ceph/cluster/mon/drain.go index a5fcf2305..94951930d 100644 --- a/pkg/operator/ceph/cluster/mon/drain.go +++ b/pkg/operator/ceph/cluster/mon/drain.go @@ -23,6 +23,7 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" policyv1beta1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -68,3 +69,31 @@ func (c *Cluster) createOrUpdateMonPDB(maxUnavailable int32) (controllerutil.Ope } return controllerutil.CreateOrUpdate(context.TODO(), c.context.Client, pdb, mutateFunc) } + +// blockMonDrain makes MaxUnavailable in mon PDB to 0 to block any voluntary mon drains +func (c *Cluster) blockMonDrain(request types.NamespacedName) error { + if !c.spec.DisruptionManagement.ManagePodBudgets { + return nil + } + logger.Info("prevent voluntary mon drain while failing over") + // change MaxUnavailable mon PDB to 0 + _, err := c.createOrUpdateMonPDB(0) + if err != nil { + return errors.Wrapf(err, "failed to update MaxUnavailable for mon PDB %q", request.Name) + } + return nil +} + +// allowMonDrain updates the MaxUnavailable in mon PDB to 1 to allow voluntary mon drains +func (c *Cluster) allowMonDrain(request types.NamespacedName) error { + if !c.spec.DisruptionManagement.ManagePodBudgets { + return nil + } + logger.Info("allow voluntary mon drain after failover") + // change MaxUnavailable mon PDB to 1 + _, err := c.createOrUpdateMonPDB(1) + if err != nil { + return errors.Wrapf(err, "failed to update MaxUnavailable for mon PDB %q", request.Name) + } + return nil +} diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 9fbe1dfff..13c11bc7b 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -30,6 +30,7 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) var ( @@ -237,10 +238,20 @@ func (c *Cluster) failMon(monCount, desiredMonCount int, name string) { logger.Errorf("failed to remove mon %q. %v", name, err) } } else { + //prevent any voluntary mon drain while failing over + if err := c.blockMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil { + logger.Errorf("failed to block mon drain. %v", err) + } + // bring up a new mon to replace the unhealthy mon if err := c.failoverMon(name); err != nil { logger.Errorf("failed to failover mon %q. %v", name, err) } + + // allow any voluntary mon drain after failover + if err := c.allowMonDrain(types.NamespacedName{Name: monPDBName, Namespace: c.Namespace}); err != nil { + logger.Errorf("failed to allow mon drain. %v", err) + } } }