diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 067b754a0..5ca5fddc6 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -247,7 +247,12 @@ A specific will contain a specific release of Ceph as well as security fixes fro ### Mon Settings -* `count`: Set the number of mons to be started. The number must be odd and between `1` and `9`. If not specified the default is set to `3`. +* `count`: Set the number of mons to be started. The number must be between `1` and `9`. The recommended value is most commonly `3`. + For highest availability, an odd number of mons should be specified. + For higher durability in case of mon loss, an even number can be specified although availability may be lower. + To maintain quorum a majority of mons must be up. For example, if there are three mons, two must be up. + If there are four mons, three must be up. If there are two mons, both must be up. + If quorum is lost, see the [disaster recovery guide](ceph-disaster-recovery.md#restoring-mon-quorum) to restore quorum from a single mon. * `allowMultiplePerNode`: Whether to allow the placement of multiple mons on a single node. Default is `false` for production. Should only be set to `true` in test environments. * `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath diff --git a/Documentation/ceph-mon-health.md b/Documentation/ceph-mon-health.md index f67cb5e65..0ae91a30b 100644 --- a/Documentation/ceph-mon-health.md +++ b/Documentation/ceph-mon-health.md @@ -34,9 +34,9 @@ quorum and perform operations in the cluster. If the majority of mons are not ru Most commonly a cluster will have three mons. This would mean that one mon could go down and allow the cluster to remain healthy. You would still have 2/3 mons running to give you consensus in the cluster for any operation. -You will always want an odd number of mons. Fifty percent of mons will not be sufficient to maintain quorum. If you had two mons and one +For highest availability, an odd number of mons is required. Fifty percent of mons will not be sufficient to maintain quorum. If you had two mons and one of them went down, you would have 1/2 of quorum. Since that is not a super-majority, the cluster would have to wait until the second mon is up again. -Therefore, Rook prohibits an even number of mons. +Rook allows an even number of mons for higher durability. See the [disaster recovery guide](ceph-disaster-recovery.md#restoring-mon-quorum) if quorum is lost and to recover mon quorum from a single mon. The number of mons to create in a cluster depends on your tolerance for losing a node. If you have 1 mon zero nodes can be lost to maintain quorum. With 3 mons one node can be lost, and with 5 mons two nodes can be lost. Because the Rook operator will automatically diff --git a/cluster/charts/rook-ceph-cluster/values.yaml b/cluster/charts/rook-ceph-cluster/values.yaml index 7e284669c..8cace5e27 100644 --- a/cluster/charts/rook-ceph-cluster/values.yaml +++ b/cluster/charts/rook-ceph-cluster/values.yaml @@ -70,7 +70,8 @@ cephClusterSpec: waitTimeoutForHealthyOSDInMinutes: 10 mon: - # Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3. + # Set the number of mons to be started. Generally recommended to be 3. + # For highest availability, an odd number of mons should be specified. count: 3 # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason. # Mons should only be allowed on the same node for test environments where data loss is acceptable. @@ -328,14 +329,14 @@ cephBlockPools: # For nbd options refer # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options # mapOptions: lock_on_read,queue_depth=1024 - + # (optional) unmapOptions is a comma-separated list of unmap options. # For krbd options refer # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options # For nbd options refer # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options # unmapOptions: force - + # RBD image format. Defaults to "2". imageFormat: "2" # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 74c8554f8..1d687d036 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -812,6 +812,7 @@ spec: type: boolean count: description: Count is the number of Ceph monitors + maximum: 9 minimum: 0 type: integer stretchCluster: diff --git a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml index eb796fc74..561334960 100644 --- a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml @@ -14,7 +14,8 @@ metadata: spec: dataDirHostPath: /var/lib/rook mon: - # Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3. + # Set the number of mons to be started. Generally recommended to be 3. + # For highest availability, an odd number of mons should be specified. count: 3 # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason. # Mons should only be allowed on the same node for test environments where data loss is acceptable. diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index cb2ac1ea7..dd2c46283 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -44,7 +44,8 @@ spec: # The default wait timeout is 10 minutes. waitTimeoutForHealthyOSDInMinutes: 10 mon: - # Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3. + # Set the number of mons to be started. Generally recommended to be 3. + # For highest availability, an odd number of mons should be specified. count: 3 # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason. # Mons should only be allowed on the same node for test environments where data loss is acceptable. diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index e7b1f64fa..dac3739ab 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -812,6 +812,7 @@ spec: type: boolean count: description: Count is the number of Ceph monitors + maximum: 9 minimum: 0 type: integer stretchCluster: diff --git a/pkg/apis/ceph.rook.io/v1/cluster.go b/pkg/apis/ceph.rook.io/v1/cluster.go index a5c7af4a2..a1d8ad48f 100644 --- a/pkg/apis/ceph.rook.io/v1/cluster.go +++ b/pkg/apis/ceph.rook.io/v1/cluster.go @@ -55,10 +55,6 @@ func (c *CephCluster) ValidateDelete() error { } func validateUpdatedCephCluster(updatedCephCluster *CephCluster, found *CephCluster) error { - if updatedCephCluster.Spec.Mon.Count > 0 && updatedCephCluster.Spec.Mon.Count%2 == 0 { - return errors.Errorf("mon count %d cannot be even, must be odd to support a healthy quorum", updatedCephCluster.Spec.Mon.Count) - } - if updatedCephCluster.Spec.DataDirHostPath != found.Spec.DataDirHostPath { return errors.Errorf("invalid update: DataDirHostPath change from %q to %q is not allowed", found.Spec.DataDirHostPath, updatedCephCluster.Spec.DataDirHostPath) } diff --git a/pkg/apis/ceph.rook.io/v1/cluster_test.go b/pkg/apis/ceph.rook.io/v1/cluster_test.go index 6fffb9ff9..dbc169f54 100644 --- a/pkg/apis/ceph.rook.io/v1/cluster_test.go +++ b/pkg/apis/ceph.rook.io/v1/cluster_test.go @@ -35,7 +35,7 @@ func Test_validateUpdatedCephCluster(t *testing.T) { }{ {"everything is ok", args{&CephCluster{}, &CephCluster{}}, false}, {"good mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 1}}}, &CephCluster{}}, false}, - {"even mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 2}}}, &CephCluster{}}, true}, + {"even mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 2}}}, &CephCluster{}}, false}, {"good mon count", args{&CephCluster{Spec: ClusterSpec{Mon: MonSpec{Count: 3}}}, &CephCluster{}}, false}, {"changed DataDirHostPath", args{&CephCluster{Spec: ClusterSpec{DataDirHostPath: "foo"}}, &CephCluster{Spec: ClusterSpec{DataDirHostPath: "bar"}}}, true}, {"changed HostNetwork", args{&CephCluster{Spec: ClusterSpec{Network: NetworkSpec{HostNetwork: false}}}, &CephCluster{Spec: ClusterSpec{Network: NetworkSpec{HostNetwork: true}}}}, true}, diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 3059f2d10..54b9d50ae 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -467,6 +467,7 @@ const ( type MonSpec struct { // Count is the number of Ceph monitors // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=9 // +optional Count int `json:"count,omitempty"` // AllowMultiplePerNode determines if we can run multiple monitors on the same node (not recommended) diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 0721b3862..eb1182d3c 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -351,9 +351,6 @@ func preClusterStartValidation(cluster *cluster) error { logger.Warningf("mon count should be at least 1, will use default value of %d", mon.DefaultMonCount) cluster.Spec.Mon.Count = mon.DefaultMonCount } - if cluster.Spec.Mon.Count%2 == 0 { - return errors.Errorf("mon count %d cannot be even, must be odd to support a healthy quorum", cluster.Spec.Mon.Count) - } if !cluster.Spec.Mon.AllowMultiplePerNode { // Check that there are enough nodes to have a chance of starting the requested number of mons nodes, err := cluster.context.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) diff --git a/pkg/operator/ceph/cluster/cluster_test.go b/pkg/operator/ceph/cluster/cluster_test.go index eb94e1d18..d121c79f6 100644 --- a/pkg/operator/ceph/cluster/cluster_test.go +++ b/pkg/operator/ceph/cluster/cluster_test.go @@ -35,7 +35,7 @@ func TestPreClusterStartValidation(t *testing.T) { wantErr bool }{ {"no settings", args{&cluster{Spec: &cephv1.ClusterSpec{}, context: &clusterd.Context{Clientset: testop.New(t, 3)}}}, false}, - {"even mons", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 2}}}}, true}, + {"even mons", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{Count: 2}}}}, false}, {"missing stretch zones", args{&cluster{context: &clusterd.Context{Clientset: testop.New(t, 3)}, Spec: &cephv1.ClusterSpec{Mon: cephv1.MonSpec{StretchCluster: &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{ {Name: "a"}, }}}}}}, true}, diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 0afdba352..eaf45094b 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -597,10 +597,6 @@ func (c *Cluster) addOrRemoveExternalMonitor(status cephclient.MonStatusResponse logger.Debugf("ClusterInfo is now Empty, refilling it from status.MonMap.Mons") monCount := len(status.MonMap.Mons) - if monCount%2 == 0 { - logger.Warningf("external cluster mon count is even (%d), should be uneven, continuing.", monCount) - } - if monCount == 1 { logger.Warning("external cluster mon count is 1, consider adding new monitors.") }