ceph: require an odd number of mons and sufficient nodes

The ceph mons should never be started with an even number in quorum.
The desired state should always be an odd number of mons to ensure
a healthy majority quorum. The operator now rejects a request for
an even number of mons.

If there are not a sufficient number of nodes for mons to be on a
unique host, the operator also returns an early error instead of
getting stuck waiting for a pending mon. This is not foolproof since
there might be fewer available nodes for the mons, but at least it
is a quick check for the common case.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2020-10-12 15:21:27 -06:00
1 parent ede7808960
commit c7e0338d5e
7 files changed
+30 -9

No files matched your search

+6 -6
View File
@@ -170,8 +170,8 @@ A specific will contain a specific release of Ceph as well as security fixes fro
### Mon Settings
* `count`: Set the number of mons to be started. The number should be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`.
* `allowMultiplePerNode`: Enable (`true`) or disable (`false`) the placement of multiple mons on one node. Default is `false`.
* `count`: Set the number of mons to be started. The number must be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`.
* `allowMultiplePerNode`: Whether to allow the placement of multiple mons on a single node. Default is `false` for production. Should only be set to `true` in test environments.
* `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs
for monitor storage. This field is optional, and when not provided, HostPath
volume mounts are used. The current set of fields from template that are used
@@ -538,7 +538,7 @@ spec:
dataDirHostPath: /var/lib/rook
mon:
count: 3
allowMultiplePerNode: true
allowMultiplePerNode: false
dashboard:
enabled: true
# cluster level storage configuration and selection
@@ -570,7 +570,7 @@ spec:
dataDirHostPath: /var/lib/rook
mon:
count: 3
allowMultiplePerNode: true
allowMultiplePerNode: false
dashboard:
enabled: true
# cluster level storage configuration and selection
@@ -611,7 +611,7 @@ spec:
dataDirHostPath: /var/lib/rook
mon:
count: 3
allowMultiplePerNode: true
allowMultiplePerNode: false
# enable the ceph dashboard for viewing cluster status
dashboard:
enabled: true
@@ -658,7 +658,7 @@ spec:
dataDirHostPath: /var/lib/rook
mon:
count: 3
allowMultiplePerNode: true
allowMultiplePerNode: false
# enable the ceph dashboard for viewing cluster status
dashboard:
enabled: true
+3 -1
View File
@@ -10,8 +10,10 @@ v1.5...
### Ceph
- Ceph mons require an odd number for a healthy quorum. An even number of mons is now disallowed.
## Features
### Ceph
* Ceph Block Pool: add mirroring support
* Ceph Block Pool: add mirroring support
@@ -17,7 +17,10 @@ metadata:
spec:
dataDirHostPath: /var/lib/rook
mon:
# Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3.
count: 3
# The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
# Mons should only be allowed on the same node for test environments where data loss is acceptable.
allowMultiplePerNode: false
# A volume claim template can be specified in which case new monitors (and
# monitors created during fail over) will construct a PVC based on the
@@ -38,9 +38,11 @@ spec:
skipUpgradeChecks: false
# Whether or not continue if PGs are not clean during an upgrade
continueUpgradeAfterChecksEvenIfNotHealthy: false
# set the amount of mons to be started
mon:
# Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3.
count: 3
# The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason.
# Mons should only be allowed on the same node for test environments where data loss is acceptable.
allowMultiplePerNode: false
mgr:
modules:
@@ -62,6 +62,10 @@ func (c *CephCluster) ValidateDelete() error {
}
func validateUpdatedCephCluster(updatedCephCluster *CephCluster, found *CephCluster) error {
if updatedCephCluster.Spec.Mon.Count > 0 && updatedCephCluster.Spec.Mon.Count%2 == 0 {
return errors.Errorf("mon count %d cannot be even, must be odd to support a healthy quorum", updatedCephCluster.Spec.Mon.Count)
}
if updatedCephCluster.Spec.DataDirHostPath != found.Spec.DataDirHostPath {
return errors.Errorf("invalid update: DataDirHostPath change from %q to %q is not allowed", found.Spec.DataDirHostPath, updatedCephCluster.Spec.DataDirHostPath)
}
@@ -33,6 +33,9 @@ func Test_validateUpdatedCephCluster(t *testing.T) {
wantErr bool
}{
{"everything is ok", args{&CephCluster{}, &CephCluster{}}, false},
{"good mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 1}}}, &CephCluster{}}, false},
{"even mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 2}}}, &CephCluster{}}, true},
{"good mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 3}}}, &CephCluster{}}, false},
{"changed DataDirHostPath", args{&CephCluster{Spec: ClusterSpec{DataDirHostPath: "foo"}}, &CephCluster{Spec: ClusterSpec{DataDirHostPath: "bar"}}}, true},
{"changed HostNetwork", args{&CephCluster{Spec: ClusterSpec{Network: NetworkSpec{HostNetwork: false}}}, &CephCluster{Spec: ClusterSpec{Network: NetworkSpec{HostNetwork: true}}}}, true},
{"changed storageClassDeviceSet encryption", args{&CephCluster{Spec: ClusterSpec{Storage: v1.StorageScopeSpec{StorageClassDeviceSets: []v1.StorageClassDeviceSet{{Name: "foo", Encrypted: false}}}}}, &CephCluster{Spec: ClusterSpec{Storage: v1.StorageScopeSpec{StorageClassDeviceSets: []v1.StorageClassDeviceSet{{Name: "foo", Encrypted: true}}}}}}, true},
+8 -1
View File
@@ -332,7 +332,14 @@ func (c *ClusterController) preClusterStartValidation(cluster *cluster, clusterO
cluster.Spec.Mon.Count = mon.DefaultMonCount
}
if cluster.Spec.Mon.Count%2 == 0 {
logger.Warningf("mon count is even (given: %d), should be uneven, continuing", cluster.Spec.Mon.Count)
return errors.Errorf("mon count %d cannot be even, must be odd to support a healthy quorum", cluster.Spec.Mon.Count)
}
if !cluster.Spec.Mon.AllowMultiplePerNode {
// Check that there are enough nodes to have a chance of starting the requested number of mons
nodes, err := c.context.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
if err == nil && len(nodes.Items) < cluster.Spec.Mon.Count {
return errors.Errorf("cannot start %d mons on %d node(s) when allowMultiplePerNode is false", cluster.Spec.Mon.Count, len(nodes.Items))
}
}
if len(cluster.Spec.Storage.Directories) != 0 {
logger.Warning("running osds on directory is not supported anymore, use devices instead.")