From 42ea2897cff022d27189502d6a00889ef58c5a43 Mon Sep 17 00:00:00 2001 From: sp98 Date: Tue, 13 Jun 2023 12:04:46 +0530 Subject: [PATCH] mon: allow changing hostNetwork settings Allow changing spec.network.hostNetwork settings on a running cluster by failing over the mons Signed-off-by: sp98 --- Documentation/CRDs/specification.md | 7 +++- .../Advanced/ceph-mon-health.md | 7 ++++ .../charts/rook-ceph/templates/resources.yaml | 4 +- deploy/examples/crds.yaml | 4 +- pkg/apis/ceph.rook.io/v1/types.go | 7 +++- pkg/operator/ceph/cluster/mon/mon.go | 19 +++++++++ pkg/operator/ceph/cluster/mon/mon_test.go | 41 ++++++++++++++++++- 7 files changed, 80 insertions(+), 9 deletions(-) diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index e7592e645..f946897f7 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -8420,7 +8420,8 @@ NetworkProviderType (Optional) -

Provider is what provides network connectivity to the cluster e.g. “host” or “multus”

+

Provider is what provides network connectivity to the cluster e.g. “host” or “multus”. +If the Provider is updated from being empty to “host” on a running cluster, then the operator will automatically fail over all the mons to apply the “host” network settings.

@@ -8492,7 +8493,9 @@ bool (Optional) -

HostNetwork to enable host network

+

HostNetwork to enable host network. +If host networking is enabled or disabled on a running cluster, then the operator will automatically fail over all the mons to +apply the new network settings.

diff --git a/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md b/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md index 169c78b6e..ce80aa6fc 100644 --- a/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md +++ b/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md @@ -115,3 +115,10 @@ $ ceph -s osd: 3 osds: 3 up (since 10m), 3 in (since 10m) [...] ``` + +## Automatic Monitor Failover + +Rook will automatically fail over the mons when the following settings are updated in the CephCluster CR: +- `spec.network.hostNetwork`: When enabled or disabled, Rook fails over all monitors, configuring them to enable or disable host networking. +- `spec.network.Provider` : When updated from being empty to "host", Rook fails over all monitors, configuring them to enable or disable host networking. +- `spec.network.multiClusterService`: When enabled or disabled, Rook fails over all monitors, configuring them to start (or stop) using service IPs compatible with the multi-cluster service. diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index dae49adf1..c188bca43 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -2232,7 +2232,7 @@ spec: description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6 type: boolean hostNetwork: - description: HostNetwork to enable host network + description: HostNetwork to enable host network. If host networking is enabled or disabled on a running cluster, then the operator will automatically fail over all the mons to apply the new network settings. type: boolean ipFamily: description: IPFamily is the single stack IPv6 or IPv4 protocol @@ -2252,7 +2252,7 @@ spec: type: boolean type: object provider: - description: Provider is what provides network connectivity to the cluster e.g. "host" or "multus" + description: Provider is what provides network connectivity to the cluster e.g. "host" or "multus". If the Provider is updated from being empty to "host" on a running cluster, then the operator will automatically fail over all the mons to apply the "host" network settings. enum: - "" - host diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index fa232557c..f381e9c3b 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -2230,7 +2230,7 @@ spec: description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6 type: boolean hostNetwork: - description: HostNetwork to enable host network + description: HostNetwork to enable host network. If host networking is enabled or disabled on a running cluster, then the operator will automatically fail over all the mons to apply the new network settings. type: boolean ipFamily: description: IPFamily is the single stack IPv6 or IPv4 protocol @@ -2250,7 +2250,7 @@ spec: type: boolean type: object provider: - description: Provider is what provides network connectivity to the cluster e.g. "host" or "multus" + description: Provider is what provides network connectivity to the cluster e.g. "host" or "multus". If the Provider is updated from being empty to "host" on a running cluster, then the operator will automatically fail over all the mons to apply the "host" network settings. enum: - "" - host diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 050977f42..70806eb7e 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -2317,7 +2317,8 @@ type SSSDSidecarAdditionalFile struct { // NetworkSpec for Ceph includes backward compatibility code // +kubebuilder:validation:XValidation:message="at least one network selector must be specified when using multus",rule="!has(self.provider) || (self.provider != 'multus' || (self.provider == 'multus' && size(self.selectors) > 0))" type NetworkSpec struct { - // Provider is what provides network connectivity to the cluster e.g. "host" or "multus" + // Provider is what provides network connectivity to the cluster e.g. "host" or "multus". + // If the Provider is updated from being empty to "host" on a running cluster, then the operator will automatically fail over all the mons to apply the "host" network settings. // +kubebuilder:validation:XValidation:message="network provider must be disabled (reverted to empty string) before a new provider is enabled",rule="self == '' || self == oldSelf" // +nullable // +optional @@ -2363,7 +2364,9 @@ type NetworkSpec struct { // +optional Connections *ConnectionsSpec `json:"connections,omitempty"` - // HostNetwork to enable host network + // HostNetwork to enable host network. + // If host networking is enabled or disabled on a running cluster, then the operator will automatically fail over all the mons to + // apply the new network settings. // +optional HostNetwork bool `json:"hostNetwork,omitempty"` diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index f3012d668..a5961793a 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -1343,6 +1343,12 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e return nil } + // skip update if mon fail over is required due to change in hostnetwork settings + if isMonIPUpdateRequiredForHostNetwork(m.DaemonName, m.UseHostNetwork, &c.spec.Network) { + c.monsToFailover.Insert(m.DaemonName) + return nil + } + // the existing deployment may have a node selector. if the cluster // isn't using host networking and the deployment is using pvc storage, // then the node selector can be removed. this may happen after @@ -1408,6 +1414,19 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e return nil } +func isMonIPUpdateRequiredForHostNetwork(mon string, isMonUsingHostNetwork bool, network *cephv1.NetworkSpec) bool { + isHostNetworkEnabledInSpec := network.IsHost() + if isHostNetworkEnabledInSpec && !isMonUsingHostNetwork { + logger.Infof("host network is enabled for the cluster but mon %q is not running on host IP address", mon) + return true + } else if !isHostNetworkEnabledInSpec && isMonUsingHostNetwork { + logger.Infof("host network is disabled for the cluster but mon %q is still running on host IP address", mon) + return true + } + + return false +} + func hasMonPathChanged(d *apps.Deployment, claim *v1.PersistentVolumeClaim) bool { if d.Labels["pvc_name"] == "" && claim != nil { logger.Infof("skipping update for mon %q where path has changed from hostPath to pvc", d.Name) diff --git a/pkg/operator/ceph/cluster/mon/mon_test.go b/pkg/operator/ceph/cluster/mon/mon_test.go index 49be715a0..c23d130e3 100644 --- a/pkg/operator/ceph/cluster/mon/mon_test.go +++ b/pkg/operator/ceph/cluster/mon/mon_test.go @@ -44,6 +44,7 @@ import ( apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" ) // generate a standard mon config from a mon id w/ default port and IP 2.4.6.{1,2,3,...} @@ -120,7 +121,8 @@ func newCluster(context *clusterd.Context, namespace string, allowMultiplePerNod mapping: &opcontroller.Mapping{ Schedule: map[string]*opcontroller.MonScheduleInfo{}, }, - ownerInfo: ownerInfo, + ownerInfo: ownerInfo, + monsToFailover: sets.New[string](), } } @@ -917,3 +919,40 @@ func TestHasMonPathChanged(t *testing.T) { assert.False(t, hasMonPathChanged(monDeployment, nil)) }) } + +func TestIsMonIPUpdateRequiredForHostNetwork(t *testing.T) { + t.Run("both cluster and mon are set to use host network", func(t *testing.T) { + hostNetwork := &cephv1.NetworkSpec{HostNetwork: true} + monUsingHostNetwork := true + assert.False(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork)) + }) + + t.Run("both cluster and mon are not set for host network", func(t *testing.T) { + hostNetwork := &cephv1.NetworkSpec{} + monUsingHostNetwork := false + assert.False(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork)) + }) + t.Run("cluster is set for host networking but mon pod is not", func(t *testing.T) { + hostNetwork := &cephv1.NetworkSpec{HostNetwork: true} + monUsingHostNetwork := false + assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork)) + }) + + t.Run("mon is using host networking but cluster is updated to not use host network ", func(t *testing.T) { + hostNetwork := &cephv1.NetworkSpec{} + monUsingHostNetwork := true + assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork)) + }) + + t.Run("mon is using host networking and cluster is set host network via NetworkProviderHost ", func(t *testing.T) { + hostNetwork := &cephv1.NetworkSpec{Provider: cephv1.NetworkProviderHost} + monUsingHostNetwork := true + assert.False(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork)) + }) + + t.Run("mon is not using host networking but cluster is updated to use host network via NetworkProviderHost ", func(t *testing.T) { + hostNetwork := &cephv1.NetworkSpec{Provider: cephv1.NetworkProviderHost} + monUsingHostNetwork := false + assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork)) + }) +}