From 19ffd749e0db3ab1198ffcfe4eb2d8e6d4865fe0 Mon Sep 17 00:00:00 2001 From: travisn Date: Tue, 12 Sep 2023 16:19:01 -0600 Subject: [PATCH] mgr: allow more than two mgrs While two mgrs is considered sufficient, more mgr daemons are now allowed in case the admin wants even more fault tolerance to the active mgr going down, in case multiple mgrs go down. Up to five mgr pods will be allowed. All mgrs will be in standby mode except one active mgr. All mgr pods have a sidecar that will update their respective pod specs with the active or passive label. Signed-off-by: travisn (cherry picked from commit 66547d27be8b2e2267ba06ecde287401f966b0d6) --- Documentation/CRDs/specification.md | 2 +- .../charts/rook-ceph/templates/resources.yaml | 4 +- deploy/examples/crds.yaml | 4 +- pkg/apis/ceph.rook.io/v1/types.go | 4 +- pkg/operator/ceph/cluster/mgr/mgr.go | 43 ++++++++++++++----- 5 files changed, 39 insertions(+), 18 deletions(-) diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index f9b8cd385..81c241715 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -7218,7 +7218,7 @@ int (Optional) -

Count is the number of manager to run

+

Count is the number of manager daemons to run

diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 5ed351b3d..c40eee724 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -1323,8 +1323,8 @@ spec: description: AllowMultiplePerNode allows to run multiple managers on the same node (not recommended) type: boolean count: - description: Count is the number of manager to run - maximum: 2 + description: Count is the number of manager daemons to run + maximum: 5 minimum: 0 type: integer modules: diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 731992302..f808d725f 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -1321,8 +1321,8 @@ spec: description: AllowMultiplePerNode allows to run multiple managers on the same node (not recommended) type: boolean count: - description: Count is the number of manager to run - maximum: 2 + description: Count is the number of manager daemons to run + maximum: 5 minimum: 0 type: integer modules: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index c4c347e66..791b4e2f5 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -581,9 +581,9 @@ type MonZoneSpec struct { // MgrSpec represents options to configure a ceph mgr type MgrSpec struct { - // Count is the number of manager to run + // Count is the number of manager daemons to run // +kubebuilder:validation:Minimum=0 - // +kubebuilder:validation:Maximum=2 + // +kubebuilder:validation:Maximum=5 // +optional Count int `json:"count,omitempty"` // AllowMultiplePerNode allows to run multiple managers on the same node (not recommended) diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index f72db4b1e..b2d99db16 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -44,7 +44,6 @@ var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mgr") const ( AppName = "rook-ceph-mgr" serviceAccountName = "rook-ceph-mgr" - maxMgrCount = 2 PrometheusModuleName = "prometheus" crashModuleName = "crash" PgautoscalerModuleName = "pg_autoscaler" @@ -97,9 +96,6 @@ func (c *Cluster) getReplicas() int { func (c *Cluster) getDaemonIDs() []string { var daemonIDs []string replicas := c.getReplicas() - if replicas > maxMgrCount { - replicas = maxMgrCount - } for i := 0; i < replicas; i++ { daemonIDs = append(daemonIDs, k8sutil.IndexToName(i)) } @@ -201,14 +197,39 @@ func (c *Cluster) Start() error { } func (c *Cluster) removeExtraMgrs(daemonIDs []string) { + options := metav1.ListOptions{LabelSelector: "app=" + AppName} + mgrDeployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, options) + if err != nil { + logger.Warningf("failed to check for extra mgrs. %v", err) + return + } + if len(mgrDeployments.Items) == len(daemonIDs) { + logger.Debugf("expected number %d of mgrs found", len(daemonIDs)) + return + } + // In case the mgr count was reduced, delete the extra mgrs - for i := maxMgrCount - 1; i >= len(daemonIDs); i-- { - mgrName := fmt.Sprintf("%s-%s", AppName, k8sutil.IndexToName(i)) - err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, mgrName, metav1.DeleteOptions{}) - if err == nil { - logger.Infof("removed extra mgr %q", mgrName) - } else if !kerrors.IsNotFound(err) { - logger.Warningf("failed to remove extra mgr %q. %v", mgrName, err) + for _, mgrDeployment := range mgrDeployments.Items { + id, ok := mgrDeployment.Labels[controller.DaemonIDLabel] + if !ok { + // skipping evaluation of non-mgr daemon that mistakenly matched the mgr labels + continue + } + found := false + for _, daemonID := range daemonIDs { + if id == daemonID { + // mark the mgr as found if the ID matches + found = true + break + } + } + if !found { + err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).Delete(c.clusterInfo.Context, mgrDeployment.Name, metav1.DeleteOptions{}) + if err == nil { + logger.Infof("removed extra mgr %q", mgrDeployment.Name) + } else { + logger.Warningf("failed to remove extra mgr %q. %v", mgrDeployment.Name, err) + } } } }