ceph: remove the CRD option Mon.PreferredCount
Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
This commit is contained in:
1 parent
94bd70a14c
commit
e2fccdf03b
11 files changed
+8
-145
No files matched your search
@@ -130,10 +130,6 @@ For more details on the mons and when to choose a number other than `3`, see the
|
||||
### Mon Settings
|
||||
|
||||
- `count`: Set the number of mons to be started. The number should be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`.
|
||||
- `preferredCount`: If you want to increase the number of mons when the number of nodes increases, set the `preferredCount` to be larger than the `count`. For example, if your cluster
|
||||
starts with three nodes, but might grow to more than five nodes, you might want five mons running after the other nodes come online. In this case, set `count: 3` and `preferredCount: 5`.
|
||||
When the operator sees the new nodes come online, the number of mons will increase to the preferred count. If the number of nodes decreases below the `preferredCount`, the operator will
|
||||
reduce the number of mons back to `count`. If `allowMultiplePerNode: true` (for testing scenarios), the number of mons will always use `preferredCount` if set.
|
||||
- `allowMultiplePerNode`: Enable (`true`) or disable (`false`) the placement of multiple mons on one node. Default is `false`.
|
||||
- `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs
|
||||
for monitor storage. This field is optional, and when not provided, HostPath
|
||||
|
||||
@@ -52,6 +52,7 @@ an example usage
|
||||
- The minimum version supported by Rook is Ceph Mimic v13.2.4. Before upgrading to v1.1 it is required to update the version of Ceph to at least this version.
|
||||
- The CSI driver is enabled by default. Documentation has been changed significantly for block and filesystem to use the CSI driver instead of flex.
|
||||
While the flex driver is still supported, it is anticipated to be deprecated soon.
|
||||
- The `Mon.PreferredCount` setting has been removed.
|
||||
|
||||
## Known Issues
|
||||
|
||||
|
||||
@@ -47,10 +47,6 @@ spec:
|
||||
maximum: 9
|
||||
minimum: 1
|
||||
type: integer
|
||||
preferredCount:
|
||||
maximum: 9
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- count
|
||||
network:
|
||||
|
||||
@@ -66,10 +66,6 @@ spec:
|
||||
maximum: 9
|
||||
minimum: 0
|
||||
type: integer
|
||||
preferredCount:
|
||||
maximum: 9
|
||||
minimum: 0
|
||||
type: integer
|
||||
network:
|
||||
properties:
|
||||
hostNetwork:
|
||||
|
||||
@@ -23,7 +23,6 @@ claim template that is used to generate PVCs for monitor database storage.
|
||||
```go
|
||||
type MonSpec struct {
|
||||
Count int `json:"count"`
|
||||
PreferredCount int `json:"preferredCount"`
|
||||
AllowMultiplePerNode bool `json:"allowMultiplePerNode"`
|
||||
VolumeClaimTemplate *v1.PersistentVolumeClaim
|
||||
}
|
||||
@@ -128,6 +127,8 @@ has eviction policies in the future we could then make use of it (e.g. with
|
||||
|
||||
### Target Monitor Count
|
||||
|
||||
The `PreferredCount` feature has been removed.
|
||||
|
||||
The CRD monitor count specifies a target minimum number of monitors to maintain.
|
||||
Additionally, a preferred count is available which will be the desired number of
|
||||
sufficient number of nodes are available. Unfortunately, this calculation
|
||||
@@ -135,6 +136,3 @@ relies on determining if monitor pods may be placed on a node, requiring
|
||||
knowledge of the scheduling policy and algorithm. The scenario to watch out for
|
||||
is an endless loop in which the health check is determining a bad placement but
|
||||
the k8s schedule thinks otherwise.
|
||||
|
||||
**TODO**: one approach here may be to schedule the desired number of canary pods
|
||||
to check for capacity before proceeding through monitor orchestration.
|
||||
@@ -150,7 +150,6 @@ const (
|
||||
|
||||
type MonSpec struct {
|
||||
Count int `json:"count,omitempty"`
|
||||
PreferredCount int `json:"preferredCount,omitempty"`
|
||||
AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"`
|
||||
VolumeClaimTemplate *v1.PersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"`
|
||||
}
|
||||
|
||||
@@ -92,11 +92,8 @@ func (c *Cluster) checkHealth() error {
|
||||
|
||||
// Use a local mon count in case the user updates the crd in another goroutine.
|
||||
// We need to complete a health check with a consistent value.
|
||||
desiredMonCount, msg, err := c.getTargetMonCount()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get target mon count. %+v", err)
|
||||
}
|
||||
logger.Debugf(msg)
|
||||
desiredMonCount := c.spec.Mon.Count
|
||||
logger.Debugf("targeting the mon count %d", desiredMonCount)
|
||||
|
||||
// Source of truth of which mons should exist is our *clusterInfo*
|
||||
monsNotFound := map[string]interface{}{}
|
||||
|
||||
@@ -202,14 +202,10 @@ func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string,
|
||||
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
|
||||
}
|
||||
|
||||
targetCount, msg, err := c.getTargetMonCount()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get target mon count. %+v", err)
|
||||
}
|
||||
logger.Infof(msg)
|
||||
logger.Infof("targeting the mon count %d", c.spec.Mon.Count)
|
||||
|
||||
// create the mons for a new cluster or ensure mons are running in an existing cluster
|
||||
return c.ClusterInfo, c.startMons(targetCount)
|
||||
return c.ClusterInfo, c.startMons(c.spec.Mon.Count)
|
||||
}
|
||||
|
||||
func (c *Cluster) startMons(targetCount int) error {
|
||||
|
||||
@@ -19,10 +19,7 @@ package mon
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
// NodeUsage is a mapping between a Node and computed metadata about the node
|
||||
@@ -36,63 +33,6 @@ type NodeUsage struct {
|
||||
MonValid bool
|
||||
}
|
||||
|
||||
// Get the number of mons that the operator should be starting
|
||||
func (c *Cluster) getTargetMonCount() (int, string, error) {
|
||||
|
||||
// Get the full list of k8s nodes
|
||||
nodeOptions := metav1.ListOptions{}
|
||||
nodeOptions.TypeMeta.Kind = "Node"
|
||||
nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("failed to get nodes. %+v", err)
|
||||
}
|
||||
|
||||
// Get the nodes where it is possible to place mons
|
||||
availableNodes := []v1.Node{}
|
||||
for _, node := range nodes.Items {
|
||||
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if valid {
|
||||
availableNodes = append(availableNodes, node)
|
||||
}
|
||||
}
|
||||
|
||||
target, msg := calcTargetMonCount(len(availableNodes), c.spec.Mon)
|
||||
return target, msg, nil
|
||||
}
|
||||
|
||||
func calcTargetMonCount(nodes int, spec cephv1.MonSpec) (int, string) {
|
||||
minTarget := spec.Count
|
||||
preferredTarget := spec.PreferredCount
|
||||
|
||||
if preferredTarget <= minTarget {
|
||||
msg := fmt.Sprintf("targeting the mon count %d", minTarget)
|
||||
return minTarget, msg
|
||||
}
|
||||
if nodes >= preferredTarget {
|
||||
msg := fmt.Sprintf("targeting the preferred mon count %d since there are %d available nodes", preferredTarget, nodes)
|
||||
return preferredTarget, msg
|
||||
}
|
||||
if spec.AllowMultiplePerNode {
|
||||
msg := fmt.Sprintf("targeting the preferred mon count %d even if not that many nodes since multiple mons are allowed per node", preferredTarget)
|
||||
return preferredTarget, msg
|
||||
}
|
||||
if nodes <= minTarget {
|
||||
msg := fmt.Sprintf("targeting the min mon count %d since there are only %d available nodes", minTarget, nodes)
|
||||
return minTarget, msg
|
||||
}
|
||||
|
||||
// There are between minTarget and preferredTarget nodes. Find an odd number for mons closest to the number of nodes.
|
||||
intermediate := nodes
|
||||
if intermediate%2 == 0 {
|
||||
// Decrease to an odd number if there are an even number of nodes
|
||||
intermediate--
|
||||
}
|
||||
msg := fmt.Sprintf("targeting the calculated mon count %d since there are %d available nodes", intermediate, nodes)
|
||||
return intermediate, msg
|
||||
}
|
||||
|
||||
// Look up the immutable node name from the hostname label
|
||||
func getNodeNameFromHostname(nodes *v1.NodeList, hostname string) (string, bool) {
|
||||
for _, node := range nodes.Items {
|
||||
|
||||
@@ -216,55 +216,3 @@ func TestGetNodeInfoFromNode(t *testing.T) {
|
||||
|
||||
assert.Equal(t, "1.1.1.1", info.Address)
|
||||
}
|
||||
|
||||
func TestTargetMonCount(t *testing.T) {
|
||||
// only a single node and min 1
|
||||
spec := cephv1.MonSpec{Count: 1, PreferredCount: 0, AllowMultiplePerNode: false}
|
||||
nodes := 1
|
||||
target, msg := calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 1, target)
|
||||
logger.Infof(msg)
|
||||
|
||||
// preferred 3 mons, but only one node available
|
||||
spec.PreferredCount = 3
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 1, target)
|
||||
logger.Infof(msg)
|
||||
|
||||
// preferred 3 mons, and 3 nodes available
|
||||
nodes = 3
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 3, target)
|
||||
logger.Infof(msg)
|
||||
|
||||
// get an intermediate odd number since we have several options between the min and the node count
|
||||
nodes = 4
|
||||
spec.Count = 1
|
||||
spec.PreferredCount = 5
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 3, target)
|
||||
logger.Infof(msg)
|
||||
nodes = 5
|
||||
spec.PreferredCount = 6
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 5, target)
|
||||
logger.Infof(msg)
|
||||
nodes = 6
|
||||
spec.PreferredCount = 7
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 5, target)
|
||||
logger.Infof(msg)
|
||||
nodes = 7
|
||||
spec.PreferredCount = 7
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 7, target)
|
||||
logger.Infof(msg)
|
||||
|
||||
// multiple allowed per node, so we always returned preferred
|
||||
nodes = 1
|
||||
spec.PreferredCount = 7
|
||||
spec.AllowMultiplePerNode = true
|
||||
target, msg = calcTargetMonCount(nodes, spec)
|
||||
assert.Equal(t, 7, target)
|
||||
logger.Infof(msg)
|
||||
}
|
||||
@@ -120,10 +120,6 @@ spec:
|
||||
maximum: 9
|
||||
minimum: 1
|
||||
type: integer
|
||||
preferredCount:
|
||||
maximum: 9
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- count
|
||||
network:
|
||||
|
||||
Reference in new issue
Block a user