ceph: require an odd number of mons and sufficient nodes
The ceph mons should never be started with an even number in quorum. The desired state should always be an odd number of mons to ensure a healthy majority quorum. The operator now rejects a request for an even number of mons. If there are not a sufficient number of nodes for mons to be on a unique host, the operator also returns an early error instead of getting stuck waiting for a pending mon. This is not foolproof since there might be fewer available nodes for the mons, but at least it is a quick check for the common case. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
7 files changed
+30
-9
No files matched your search
@@ -170,8 +170,8 @@ A specific will contain a specific release of Ceph as well as security fixes fro
|
||||
|
||||
### Mon Settings
|
||||
|
||||
* `count`: Set the number of mons to be started. The number should be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`.
|
||||
* `allowMultiplePerNode`: Enable (`true`) or disable (`false`) the placement of multiple mons on one node. Default is `false`.
|
||||
* `count`: Set the number of mons to be started. The number must be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`.
|
||||
* `allowMultiplePerNode`: Whether to allow the placement of multiple mons on a single node. Default is `false` for production. Should only be set to `true` in test environments.
|
||||
* `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs
|
||||
for monitor storage. This field is optional, and when not provided, HostPath
|
||||
volume mounts are used. The current set of fields from template that are used
|
||||
@@ -538,7 +538,7 @@ spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
mon:
|
||||
count: 3
|
||||
allowMultiplePerNode: true
|
||||
allowMultiplePerNode: false
|
||||
dashboard:
|
||||
enabled: true
|
||||
# cluster level storage configuration and selection
|
||||
@@ -570,7 +570,7 @@ spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
mon:
|
||||
count: 3
|
||||
allowMultiplePerNode: true
|
||||
allowMultiplePerNode: false
|
||||
dashboard:
|
||||
enabled: true
|
||||
# cluster level storage configuration and selection
|
||||
@@ -611,7 +611,7 @@ spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
mon:
|
||||
count: 3
|
||||
allowMultiplePerNode: true
|
||||
allowMultiplePerNode: false
|
||||
# enable the ceph dashboard for viewing cluster status
|
||||
dashboard:
|
||||
enabled: true
|
||||
@@ -658,7 +658,7 @@ spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
mon:
|
||||
count: 3
|
||||
allowMultiplePerNode: true
|
||||
allowMultiplePerNode: false
|
||||
# enable the ceph dashboard for viewing cluster status
|
||||
dashboard:
|
||||
enabled: true
|
||||
|
||||
@@ -10,8 +10,10 @@ v1.5...
|
||||
|
||||
### Ceph
|
||||
|
||||
- Ceph mons require an odd number for a healthy quorum. An even number of mons is now disallowed.
|
||||
|
||||
## Features
|
||||
|
||||
### Ceph
|
||||
|
||||
* Ceph Block Pool: add mirroring support
|
||||
* Ceph Block Pool: add mirroring support
|
||||
@@ -17,7 +17,10 @@ metadata:
|
||||
spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
mon:
|
||||
# Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3.
|
||||
count: 3
|
||||
# The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
|
||||
# Mons should only be allowed on the same node for test environments where data loss is acceptable.
|
||||
allowMultiplePerNode: false
|
||||
# A volume claim template can be specified in which case new monitors (and
|
||||
# monitors created during fail over) will construct a PVC based on the
|
||||
|
||||
@@ -38,9 +38,11 @@ spec:
|
||||
skipUpgradeChecks: false
|
||||
# Whether or not continue if PGs are not clean during an upgrade
|
||||
continueUpgradeAfterChecksEvenIfNotHealthy: false
|
||||
# set the amount of mons to be started
|
||||
mon:
|
||||
# Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3.
|
||||
count: 3
|
||||
# The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
|
||||
# Mons should only be allowed on the same node for test environments where data loss is acceptable.
|
||||
allowMultiplePerNode: false
|
||||
mgr:
|
||||
modules:
|
||||
|
||||
@@ -62,6 +62,10 @@ func (c *CephCluster) ValidateDelete() error {
|
||||
}
|
||||
|
||||
func validateUpdatedCephCluster(updatedCephCluster *CephCluster, found *CephCluster) error {
|
||||
if updatedCephCluster.Spec.Mon.Count > 0 && updatedCephCluster.Spec.Mon.Count%2 == 0 {
|
||||
return errors.Errorf("mon count %d cannot be even, must be odd to support a healthy quorum", updatedCephCluster.Spec.Mon.Count)
|
||||
}
|
||||
|
||||
if updatedCephCluster.Spec.DataDirHostPath != found.Spec.DataDirHostPath {
|
||||
return errors.Errorf("invalid update: DataDirHostPath change from %q to %q is not allowed", found.Spec.DataDirHostPath, updatedCephCluster.Spec.DataDirHostPath)
|
||||
}
|
||||
|
||||
@@ -33,6 +33,9 @@ func Test_validateUpdatedCephCluster(t *testing.T) {
|
||||
wantErr bool
|
||||
}{
|
||||
{"everything is ok", args{&CephCluster{}, &CephCluster{}}, false},
|
||||
{"good mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 1}}}, &CephCluster{}}, false},
|
||||
{"even mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 2}}}, &CephCluster{}}, true},
|
||||
{"good mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 3}}}, &CephCluster{}}, false},
|
||||
{"changed DataDirHostPath", args{&CephCluster{Spec: ClusterSpec{DataDirHostPath: "foo"}}, &CephCluster{Spec: ClusterSpec{DataDirHostPath: "bar"}}}, true},
|
||||
{"changed HostNetwork", args{&CephCluster{Spec: ClusterSpec{Network: NetworkSpec{HostNetwork: false}}}, &CephCluster{Spec: ClusterSpec{Network: NetworkSpec{HostNetwork: true}}}}, true},
|
||||
{"changed storageClassDeviceSet encryption", args{&CephCluster{Spec: ClusterSpec{Storage: v1.StorageScopeSpec{StorageClassDeviceSets: []v1.StorageClassDeviceSet{{Name: "foo", Encrypted: false}}}}}, &CephCluster{Spec: ClusterSpec{Storage: v1.StorageScopeSpec{StorageClassDeviceSets: []v1.StorageClassDeviceSet{{Name: "foo", Encrypted: true}}}}}}, true},
|
||||
|
||||
@@ -332,7 +332,14 @@ func (c *ClusterController) preClusterStartValidation(cluster *cluster, clusterO
|
||||
cluster.Spec.Mon.Count = mon.DefaultMonCount
|
||||
}
|
||||
if cluster.Spec.Mon.Count%2 == 0 {
|
||||
logger.Warningf("mon count is even (given: %d), should be uneven, continuing", cluster.Spec.Mon.Count)
|
||||
return errors.Errorf("mon count %d cannot be even, must be odd to support a healthy quorum", cluster.Spec.Mon.Count)
|
||||
}
|
||||
if !cluster.Spec.Mon.AllowMultiplePerNode {
|
||||
// Check that there are enough nodes to have a chance of starting the requested number of mons
|
||||
nodes, err := c.context.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
|
||||
if err == nil && len(nodes.Items) < cluster.Spec.Mon.Count {
|
||||
return errors.Errorf("cannot start %d mons on %d node(s) when allowMultiplePerNode is false", cluster.Spec.Mon.Count, len(nodes.Items))
|
||||
}
|
||||
}
|
||||
if len(cluster.Spec.Storage.Directories) != 0 {
|
||||
logger.Warning("running osds on directory is not supported anymore, use devices instead.")
|
||||
|
||||
Reference in new issue
Block a user