From c7e0338d5e90fd7bedf32cb46f3997e4d09e85ab Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 9 Oct 2020 14:56:46 -0600 Subject: [PATCH] ceph: require an odd number of mons and sufficient nodes The ceph mons should never be started with an even number in quorum. The desired state should always be an odd number of mons to ensure a healthy majority quorum. The operator now rejects a request for an even number of mons. If there are not a sufficient number of nodes for mons to be on a unique host, the operator also returns an early error instead of getting stuck waiting for a pending mon. This is not foolproof since there might be fewer available nodes for the mons, but at least it is a quick check for the common case. Signed-off-by: Travis Nielsen --- Documentation/ceph-cluster-crd.md | 12 ++++++------ PendingReleaseNotes.md | 4 +++- cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml | 3 +++ cluster/examples/kubernetes/ceph/cluster.yaml | 4 +++- pkg/apis/ceph.rook.io/v1/validate_clusters.go | 4 ++++ pkg/apis/ceph.rook.io/v1/validate_clusters_test.go | 3 +++ pkg/operator/ceph/cluster/cluster.go | 9 ++++++++- 7 files changed, 30 insertions(+), 9 deletions(-) diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index ab1675b02..2f15f8078 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -170,8 +170,8 @@ A specific will contain a specific release of Ceph as well as security fixes fro ### Mon Settings -* `count`: Set the number of mons to be started. The number should be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`. -* `allowMultiplePerNode`: Enable (`true`) or disable (`false`) the placement of multiple mons on one node. Default is `false`. +* `count`: Set the number of mons to be started. The number must be odd and between `1` and `9`. If not specified the default is set to `3` and `allowMultiplePerNode` is also set to `true`. +* `allowMultiplePerNode`: Whether to allow the placement of multiple mons on a single node. Default is `false` for production. Should only be set to `true` in test environments. * `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath volume mounts are used. The current set of fields from template that are used @@ -538,7 +538,7 @@ spec: dataDirHostPath: /var/lib/rook mon: count: 3 - allowMultiplePerNode: true + allowMultiplePerNode: false dashboard: enabled: true # cluster level storage configuration and selection @@ -570,7 +570,7 @@ spec: dataDirHostPath: /var/lib/rook mon: count: 3 - allowMultiplePerNode: true + allowMultiplePerNode: false dashboard: enabled: true # cluster level storage configuration and selection @@ -611,7 +611,7 @@ spec: dataDirHostPath: /var/lib/rook mon: count: 3 - allowMultiplePerNode: true + allowMultiplePerNode: false # enable the ceph dashboard for viewing cluster status dashboard: enabled: true @@ -658,7 +658,7 @@ spec: dataDirHostPath: /var/lib/rook mon: count: 3 - allowMultiplePerNode: true + allowMultiplePerNode: false # enable the ceph dashboard for viewing cluster status dashboard: enabled: true diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 52727e4f7..3a5b4b470 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -10,8 +10,10 @@ v1.5... ### Ceph +- Ceph mons require an odd number for a healthy quorum. An even number of mons is now disallowed. + ## Features ### Ceph -* Ceph Block Pool: add mirroring support \ No newline at end of file +* Ceph Block Pool: add mirroring support diff --git a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml index f383ed17e..a7bf16bc4 100644 --- a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml @@ -17,7 +17,10 @@ metadata: spec: dataDirHostPath: /var/lib/rook mon: + # Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3. count: 3 + # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason. + # Mons should only be allowed on the same node for test environments where data loss is acceptable. allowMultiplePerNode: false # A volume claim template can be specified in which case new monitors (and # monitors created during fail over) will construct a PVC based on the diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index a8dc47e4f..8108b7209 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -38,9 +38,11 @@ spec: skipUpgradeChecks: false # Whether or not continue if PGs are not clean during an upgrade continueUpgradeAfterChecksEvenIfNotHealthy: false - # set the amount of mons to be started mon: + # Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3. count: 3 + # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason. + # Mons should only be allowed on the same node for test environments where data loss is acceptable. allowMultiplePerNode: false mgr: modules: diff --git a/pkg/apis/ceph.rook.io/v1/validate_clusters.go b/pkg/apis/ceph.rook.io/v1/validate_clusters.go index 50b59ff8b..ce5a87c9a 100644 --- a/pkg/apis/ceph.rook.io/v1/validate_clusters.go +++ b/pkg/apis/ceph.rook.io/v1/validate_clusters.go @@ -62,6 +62,10 @@ func (c *CephCluster) ValidateDelete() error { } func validateUpdatedCephCluster(updatedCephCluster *CephCluster, found *CephCluster) error { + if updatedCephCluster.Spec.Mon.Count > 0 && updatedCephCluster.Spec.Mon.Count%2 == 0 { + return errors.Errorf("mon count %d cannot be even, must be odd to support a healthy quorum", updatedCephCluster.Spec.Mon.Count) + } + if updatedCephCluster.Spec.DataDirHostPath != found.Spec.DataDirHostPath { return errors.Errorf("invalid update: DataDirHostPath change from %q to %q is not allowed", found.Spec.DataDirHostPath, updatedCephCluster.Spec.DataDirHostPath) } diff --git a/pkg/apis/ceph.rook.io/v1/validate_clusters_test.go b/pkg/apis/ceph.rook.io/v1/validate_clusters_test.go index 472c4eb5a..4cec789a7 100644 --- a/pkg/apis/ceph.rook.io/v1/validate_clusters_test.go +++ b/pkg/apis/ceph.rook.io/v1/validate_clusters_test.go @@ -33,6 +33,9 @@ func Test_validateUpdatedCephCluster(t *testing.T) { wantErr bool }{ {"everything is ok", args{&CephCluster{}, &CephCluster{}}, false}, + {"good mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 1}}}, &CephCluster{}}, false}, + {"even mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 2}}}, &CephCluster{}}, true}, + {"good mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 3}}}, &CephCluster{}}, false}, {"changed DataDirHostPath", args{&CephCluster{Spec: ClusterSpec{DataDirHostPath: "foo"}}, &CephCluster{Spec: ClusterSpec{DataDirHostPath: "bar"}}}, true}, {"changed HostNetwork", args{&CephCluster{Spec: ClusterSpec{Network: NetworkSpec{HostNetwork: false}}}, &CephCluster{Spec: ClusterSpec{Network: NetworkSpec{HostNetwork: true}}}}, true}, {"changed storageClassDeviceSet encryption", args{&CephCluster{Spec: ClusterSpec{Storage: v1.StorageScopeSpec{StorageClassDeviceSets: []v1.StorageClassDeviceSet{{Name: "foo", Encrypted: false}}}}}, &CephCluster{Spec: ClusterSpec{Storage: v1.StorageScopeSpec{StorageClassDeviceSets: []v1.StorageClassDeviceSet{{Name: "foo", Encrypted: true}}}}}}, true}, diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 218031b9f..c8a6ae913 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -332,7 +332,14 @@ func (c *ClusterController) preClusterStartValidation(cluster *cluster, clusterO cluster.Spec.Mon.Count = mon.DefaultMonCount } if cluster.Spec.Mon.Count%2 == 0 { - logger.Warningf("mon count is even (given: %d), should be uneven, continuing", cluster.Spec.Mon.Count) + return errors.Errorf("mon count %d cannot be even, must be odd to support a healthy quorum", cluster.Spec.Mon.Count) + } + if !cluster.Spec.Mon.AllowMultiplePerNode { + // Check that there are enough nodes to have a chance of starting the requested number of mons + nodes, err := c.context.Clientset.CoreV1().Nodes().List(metav1.ListOptions{}) + if err == nil && len(nodes.Items) < cluster.Spec.Mon.Count { + return errors.Errorf("cannot start %d mons on %d node(s) when allowMultiplePerNode is false", cluster.Spec.Mon.Count, len(nodes.Items)) + } } if len(cluster.Spec.Storage.Directories) != 0 { logger.Warning("running osds on directory is not supported anymore, use devices instead.")