ceph: remove the CRD option Mon.PreferredCount

Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
This commit is contained in:
Noah Watkins committed 2019-08-27 17:43:35 -07:00
1 parent 94bd70a14c
commit e2fccdf03b
11 files changed
+8 -145

No files matched your search

-4
View File
@@ -130,10 +130,6 @@ For more details on the mons and when to choose a number other than `3`, see the
### Mon Settings
- `count`: Set the number of mons to be started. The number should be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`.
- `preferredCount`: If you want to increase the number of mons when the number of nodes increases, set the `preferredCount` to be larger than the `count`. For example, if your cluster
starts with three nodes, but might grow to more than five nodes, you might want five mons running after the other nodes come online. In this case, set `count: 3` and `preferredCount: 5`.
When the operator sees the new nodes come online, the number of mons will increase to the preferred count. If the number of nodes decreases below the `preferredCount`, the operator will
reduce the number of mons back to `count`. If `allowMultiplePerNode: true` (for testing scenarios), the number of mons will always use `preferredCount` if set.
- `allowMultiplePerNode`: Enable (`true`) or disable (`false`) the placement of multiple mons on one node. Default is `false`.
- `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs
for monitor storage. This field is optional, and when not provided, HostPath
+1
View File
@@ -52,6 +52,7 @@ an example usage
- The minimum version supported by Rook is Ceph Mimic v13.2.4. Before upgrading to v1.1 it is required to update the version of Ceph to at least this version.
- The CSI driver is enabled by default. Documentation has been changed significantly for block and filesystem to use the CSI driver instead of flex.
While the flex driver is still supported, it is anticipated to be deprecated soon.
- The `Mon.PreferredCount` setting has been removed.
## Known Issues
@@ -47,10 +47,6 @@ spec:
maximum: 9
minimum: 1
type: integer
preferredCount:
maximum: 9
minimum: 0
type: integer
required:
- count
network:
@@ -66,10 +66,6 @@ spec:
maximum: 9
minimum: 0
type: integer
preferredCount:
maximum: 9
minimum: 0
type: integer
network:
properties:
hostNetwork:
+2 -4
View File
@@ -23,7 +23,6 @@ claim template that is used to generate PVCs for monitor database storage.
```go
type MonSpec struct {
Count int `json:"count"`
PreferredCount int `json:"preferredCount"`
AllowMultiplePerNode bool `json:"allowMultiplePerNode"`
VolumeClaimTemplate *v1.PersistentVolumeClaim
}
@@ -128,6 +127,8 @@ has eviction policies in the future we could then make use of it (e.g. with
### Target Monitor Count
The `PreferredCount` feature has been removed.
The CRD monitor count specifies a target minimum number of monitors to maintain.
Additionally, a preferred count is available which will be the desired number of
sufficient number of nodes are available. Unfortunately, this calculation
@@ -135,6 +136,3 @@ relies on determining if monitor pods may be placed on a node, requiring
knowledge of the scheduling policy and algorithm. The scenario to watch out for
is an endless loop in which the health check is determining a bad placement but
the k8s schedule thinks otherwise.
**TODO**: one approach here may be to schedule the desired number of canary pods
to check for capacity before proceeding through monitor orchestration.
-1
View File
@@ -150,7 +150,6 @@ const (
type MonSpec struct {
Count int `json:"count,omitempty"`
PreferredCount int `json:"preferredCount,omitempty"`
AllowMultiplePerNode bool `json:"allowMultiplePerNode,omitempty"`
VolumeClaimTemplate *v1.PersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"`
}
+2 -5
View File
@@ -92,11 +92,8 @@ func (c *Cluster) checkHealth() error {
// Use a local mon count in case the user updates the crd in another goroutine.
// We need to complete a health check with a consistent value.
desiredMonCount, msg, err := c.getTargetMonCount()
if err != nil {
return fmt.Errorf("failed to get target mon count. %+v", err)
}
logger.Debugf(msg)
desiredMonCount := c.spec.Mon.Count
logger.Debugf("targeting the mon count %d", desiredMonCount)
// Source of truth of which mons should exist is our *clusterInfo*
monsNotFound := map[string]interface{}{}
+2 -6
View File
@@ -202,14 +202,10 @@ func (c *Cluster) Start(clusterInfo *cephconfig.ClusterInfo, rookVersion string,
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
}
targetCount, msg, err := c.getTargetMonCount()
if err != nil {
return nil, fmt.Errorf("failed to get target mon count. %+v", err)
}
logger.Infof(msg)
logger.Infof("targeting the mon count %d", c.spec.Mon.Count)
// create the mons for a new cluster or ensure mons are running in an existing cluster
return c.ClusterInfo, c.startMons(targetCount)
return c.ClusterInfo, c.startMons(c.spec.Mon.Count)
}
func (c *Cluster) startMons(targetCount int) error {
+1 -61
View File
@@ -19,10 +19,7 @@ package mon
import (
"fmt"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/operator/k8sutil"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/api/core/v1"
)
// NodeUsage is a mapping between a Node and computed metadata about the node
@@ -36,63 +33,6 @@ type NodeUsage struct {
MonValid bool
}
// Get the number of mons that the operator should be starting
func (c *Cluster) getTargetMonCount() (int, string, error) {
// Get the full list of k8s nodes
nodeOptions := metav1.ListOptions{}
nodeOptions.TypeMeta.Kind = "Node"
nodes, err := c.context.Clientset.CoreV1().Nodes().List(nodeOptions)
if err != nil {
return 0, "", fmt.Errorf("failed to get nodes. %+v", err)
}
// Get the nodes where it is possible to place mons
availableNodes := []v1.Node{}
for _, node := range nodes.Items {
valid, err := k8sutil.ValidNode(node, cephv1.GetMonPlacement(c.spec.Placement))
if err != nil {
logger.Warning("failed to validate node %s %v", node.Name, err)
} else if valid {
availableNodes = append(availableNodes, node)
}
}
target, msg := calcTargetMonCount(len(availableNodes), c.spec.Mon)
return target, msg, nil
}
func calcTargetMonCount(nodes int, spec cephv1.MonSpec) (int, string) {
minTarget := spec.Count
preferredTarget := spec.PreferredCount
if preferredTarget <= minTarget {
msg := fmt.Sprintf("targeting the mon count %d", minTarget)
return minTarget, msg
}
if nodes >= preferredTarget {
msg := fmt.Sprintf("targeting the preferred mon count %d since there are %d available nodes", preferredTarget, nodes)
return preferredTarget, msg
}
if spec.AllowMultiplePerNode {
msg := fmt.Sprintf("targeting the preferred mon count %d even if not that many nodes since multiple mons are allowed per node", preferredTarget)
return preferredTarget, msg
}
if nodes <= minTarget {
msg := fmt.Sprintf("targeting the min mon count %d since there are only %d available nodes", minTarget, nodes)
return minTarget, msg
}
// There are between minTarget and preferredTarget nodes. Find an odd number for mons closest to the number of nodes.
intermediate := nodes
if intermediate%2 == 0 {
// Decrease to an odd number if there are an even number of nodes
intermediate--
}
msg := fmt.Sprintf("targeting the calculated mon count %d since there are %d available nodes", intermediate, nodes)
return intermediate, msg
}
// Look up the immutable node name from the hostname label
func getNodeNameFromHostname(nodes *v1.NodeList, hostname string) (string, bool) {
for _, node := range nodes.Items {
@@ -216,55 +216,3 @@ func TestGetNodeInfoFromNode(t *testing.T) {
assert.Equal(t, "1.1.1.1", info.Address)
}
func TestTargetMonCount(t *testing.T) {
// only a single node and min 1
spec := cephv1.MonSpec{Count: 1, PreferredCount: 0, AllowMultiplePerNode: false}
nodes := 1
target, msg := calcTargetMonCount(nodes, spec)
assert.Equal(t, 1, target)
logger.Infof(msg)
// preferred 3 mons, but only one node available
spec.PreferredCount = 3
target, msg = calcTargetMonCount(nodes, spec)
assert.Equal(t, 1, target)
logger.Infof(msg)
// preferred 3 mons, and 3 nodes available
nodes = 3
target, msg = calcTargetMonCount(nodes, spec)
assert.Equal(t, 3, target)
logger.Infof(msg)
// get an intermediate odd number since we have several options between the min and the node count
nodes = 4
spec.Count = 1
spec.PreferredCount = 5
target, msg = calcTargetMonCount(nodes, spec)
assert.Equal(t, 3, target)
logger.Infof(msg)
nodes = 5
spec.PreferredCount = 6
target, msg = calcTargetMonCount(nodes, spec)
assert.Equal(t, 5, target)
logger.Infof(msg)
nodes = 6
spec.PreferredCount = 7
target, msg = calcTargetMonCount(nodes, spec)
assert.Equal(t, 5, target)
logger.Infof(msg)
nodes = 7
spec.PreferredCount = 7
target, msg = calcTargetMonCount(nodes, spec)
assert.Equal(t, 7, target)
logger.Infof(msg)
// multiple allowed per node, so we always returned preferred
nodes = 1
spec.PreferredCount = 7
spec.AllowMultiplePerNode = true
target, msg = calcTargetMonCount(nodes, spec)
assert.Equal(t, 7, target)
logger.Infof(msg)
}
@@ -120,10 +120,6 @@ spec:
maximum: 9
minimum: 1
type: integer
preferredCount:
maximum: 9
minimum: 0
type: integer
required:
- count
network: