diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 6559f6ccc..08c6a44d4 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -130,10 +130,6 @@ For more details on the mons and when to choose a number other than `3`, see the ### Mon Settings - `count`: Set the number of mons to be started. The number should be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`. -- `preferredCount`: If you want to increase the number of mons when the number of nodes increases, set the `preferredCount` to be larger than the `count`. For example, if your cluster -starts with three nodes, but might grow to more than five nodes, you might want five mons running after the other nodes come online. In this case, set `count: 3` and `preferredCount: 5`. -When the operator sees the new nodes come online, the number of mons will increase to the preferred count. If the number of nodes decreases below the `preferredCount`, the operator will -reduce the number of mons back to `count`. If `allowMultiplePerNode: true` (for testing scenarios), the number of mons will always use `preferredCount` if set. - `allowMultiplePerNode`: Enable (`true`) or disable (`false`) the placement of multiple mons on one node. Default is `false`. - `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 444ecd860..2300bc174 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -52,6 +52,7 @@ an example usage - The minimum version supported by Rook is Ceph Mimic v13.2.4. Before upgrading to v1.1 it is required to update the version of Ceph to at least this version. - The CSI driver is enabled by default. Documentation has been changed significantly for block and filesystem to use the CSI driver instead of flex. While the flex driver is still supported, it is anticipated to be deprecated soon. +- The `Mon.PreferredCount` setting has been removed. ## Known Issues diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 71027a620..a8c0fa7c7 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -47,10 +47,6 @@ spec: maximum: 9 minimum: 1 type: integer - preferredCount: - maximum: 9 - minimum: 0 - type: integer required: - count network: diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 1c9c66895..59cb07cc6 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -66,10 +66,6 @@ spec: maximum: 9 minimum: 0 type: integer - preferredCount: - maximum: 9 - minimum: 0 - type: integer network: properties: hostNetwork: diff --git a/design/ceph-mon-pv.md b/design/ceph-mon-pv.md index 9a513deaf..8599fe415 100644 --- a/design/ceph-mon-pv.md +++ b/design/ceph-mon-pv.md @@ -23,7 +23,6 @@ claim template that is used to generate PVCs for monitor database storage. ```go type MonSpec struct { Count int `json:"count"` - PreferredCount int `json:"preferredCount"` AllowMultiplePerNode bool `json:"allowMultiplePerNode"` VolumeClaimTemplate *v1.PersistentVolumeClaim } @@ -128,6 +127,8 @@ has eviction policies in the future we could then make use of it (e.g. with ### Target Monitor Count +The `PreferredCount` feature has been removed. + The CRD monitor count specifies a target minimum number of monitors to maintain. Additionally, a preferred count is available which will be the desired number of sufficient number of nodes are available. Unfortunately, this calculation @@ -135,6 +136,3 @@ relies on determining if monitor pods may be placed on a node, requiring knowledge of the scheduling policy and algorithm. The scenario to watch out for is an endless loop in which the health check is determining a bad placement but the k8s schedule thinks otherwise. - -**TODO**: one approach here may be to schedule the desired number of canary pods -to check for capacity before proceeding through monitor orchestration. diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 6739cca79..f2cbfa000 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -150,7 +150,6 @@ const ( type MonSpec struct { Count int `json:"count,omitempty"` - PreferredCount int `json:"preferredCount,omitempty"` AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"` VolumeClaimTemplate *v1.PersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"` } diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index d08f2a55f..e47eab881 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -92,11 +92,8 @@ func (c *Cluster) checkHealth() error { // Use a local mon count in case the user updates the crd in another goroutine. // We need to complete a health check with a consistent value. - desiredMonCount, msg, err := c.getTargetMonCount() - if err != nil { - return fmt.Errorf("failed to get target mon count. %+v", err) - } - logger.Debugf(msg) + desiredMonCount := c.spec.Mon.Count + logger.Debugf("targeting the mon count %d", desiredMonCount) // Source of truth of which mons should exist is our *clusterInfo* monsNotFound := map[string]interface{}{} diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index e0099ac1b..aa53dc755 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -202,14 +202,10 @@ func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string, return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err) } - targetCount, msg, err := c.getTargetMonCount() - if err != nil { - return nil, fmt.Errorf("failed to get target mon count. %+v", err) - } - logger.Infof(msg) + logger.Infof("targeting the mon count %d", c.spec.Mon.Count) // create the mons for a new cluster or ensure mons are running in an existing cluster - return c.ClusterInfo, c.startMons(targetCount) + return c.ClusterInfo, c.startMons(c.spec.Mon.Count) } func (c *Cluster) startMons(targetCount int) error { diff --git a/pkg/operator/ceph/cluster/mon/node.go b/pkg/operator/ceph/cluster/mon/node.go index 6b33d28ef..2bee58df1 100644 --- a/pkg/operator/ceph/cluster/mon/node.go +++ b/pkg/operator/ceph/cluster/mon/node.go @@ -19,10 +19,7 @@ package mon import ( "fmt" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/operator/k8sutil" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/api/core/v1" ) // NodeUsage is a mapping between a Node and computed metadata about the node @@ -36,63 +33,6 @@ type NodeUsage struct { MonValid bool } -// Get the number of mons that the operator should be starting -func (c *Cluster) getTargetMonCount() (int, string, error) { - - // Get the full list of k8s nodes - nodeOptions := metav1.ListOptions{} - nodeOptions.TypeMeta.Kind = "Node" - nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions) - if err != nil { - return 0, "", fmt.Errorf("failed to get nodes. %+v", err) - } - - // Get the nodes where it is possible to place mons - availableNodes := []v1.Node{} - for _, node := range nodes.Items { - valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement)) - if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) - } else if valid { - availableNodes = append(availableNodes, node) - } - } - - target, msg := calcTargetMonCount(len(availableNodes), c.spec.Mon) - return target, msg, nil -} - -func calcTargetMonCount(nodes int, spec cephv1.MonSpec) (int, string) { - minTarget := spec.Count - preferredTarget := spec.PreferredCount - - if preferredTarget <= minTarget { - msg := fmt.Sprintf("targeting the mon count %d", minTarget) - return minTarget, msg - } - if nodes >= preferredTarget { - msg := fmt.Sprintf("targeting the preferred mon count %d since there are %d available nodes", preferredTarget, nodes) - return preferredTarget, msg - } - if spec.AllowMultiplePerNode { - msg := fmt.Sprintf("targeting the preferred mon count %d even if not that many nodes since multiple mons are allowed per node", preferredTarget) - return preferredTarget, msg - } - if nodes <= minTarget { - msg := fmt.Sprintf("targeting the min mon count %d since there are only %d available nodes", minTarget, nodes) - return minTarget, msg - } - - // There are between minTarget and preferredTarget nodes. Find an odd number for mons closest to the number of nodes. - intermediate := nodes - if intermediate%2 == 0 { - // Decrease to an odd number if there are an even number of nodes - intermediate-- - } - msg := fmt.Sprintf("targeting the calculated mon count %d since there are %d available nodes", intermediate, nodes) - return intermediate, msg -} - // Look up the immutable node name from the hostname label func getNodeNameFromHostname(nodes *v1.NodeList, hostname string) (string, bool) { for _, node := range nodes.Items { diff --git a/pkg/operator/ceph/cluster/mon/node_test.go b/pkg/operator/ceph/cluster/mon/node_test.go index 7df406102..c671847db 100644 --- a/pkg/operator/ceph/cluster/mon/node_test.go +++ b/pkg/operator/ceph/cluster/mon/node_test.go @@ -216,55 +216,3 @@ func TestGetNodeInfoFromNode(t *testing.T) { assert.Equal(t, "1.1.1.1", info.Address) } - -func TestTargetMonCount(t *testing.T) { - // only a single node and min 1 - spec := cephv1.MonSpec{Count: 1, PreferredCount: 0, AllowMultiplePerNode: false} - nodes := 1 - target, msg := calcTargetMonCount(nodes, spec) - assert.Equal(t, 1, target) - logger.Infof(msg) - - // preferred 3 mons, but only one node available - spec.PreferredCount = 3 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 1, target) - logger.Infof(msg) - - // preferred 3 mons, and 3 nodes available - nodes = 3 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 3, target) - logger.Infof(msg) - - // get an intermediate odd number since we have several options between the min and the node count - nodes = 4 - spec.Count = 1 - spec.PreferredCount = 5 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 3, target) - logger.Infof(msg) - nodes = 5 - spec.PreferredCount = 6 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 5, target) - logger.Infof(msg) - nodes = 6 - spec.PreferredCount = 7 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 5, target) - logger.Infof(msg) - nodes = 7 - spec.PreferredCount = 7 - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 7, target) - logger.Infof(msg) - - // multiple allowed per node, so we always returned preferred - nodes = 1 - spec.PreferredCount = 7 - spec.AllowMultiplePerNode = true - target, msg = calcTargetMonCount(nodes, spec) - assert.Equal(t, 7, target) - logger.Infof(msg) -} diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 8c9448b6d..e0ae986ba 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -120,10 +120,6 @@ spec: maximum: 9 minimum: 1 type: integer - preferredCount: - maximum: 9 - minimum: 0 - type: integer required: - count network: