Merge pull request #12369 from sp98/mon-failover

mon: allow changing hostNetwork settings
This commit is contained in:
Blaine Gardner authored and GitHub committed 2023-12-18 08:54:59 -07:00
commit 61d0922010
7 files changed
+80 -9

No files matched your search

+5 -2
View File
@@ -8420,7 +8420,8 @@ NetworkProviderType
</td>
<td>
<em>(Optional)</em>
<p>Provider is what provides network connectivity to the cluster e.g. &ldquo;host&rdquo; or &ldquo;multus&rdquo;</p>
<p>Provider is what provides network connectivity to the cluster e.g. &ldquo;host&rdquo; or &ldquo;multus&rdquo;.
If the Provider is updated from being empty to &ldquo;host&rdquo; on a running cluster, then the operator will automatically fail over all the mons to apply the &ldquo;host&rdquo; network settings.</p>
</td>
</tr>
<tr>
@@ -8492,7 +8493,9 @@ bool
</td>
<td>
<em>(Optional)</em>
<p>HostNetwork to enable host network</p>
<p>HostNetwork to enable host network.
If host networking is enabled or disabled on a running cluster, then the operator will automatically fail over all the mons to
apply the new network settings.</p>
</td>
</tr>
<tr>
@@ -115,3 +115,10 @@ $ ceph -s
osd: 3 osds: 3 up (since 10m), 3 in (since 10m)
[...]
```
## Automatic Monitor Failover
Rook will automatically fail over the mons when the following settings are updated in the CephCluster CR:
- `spec.network.hostNetwork`: When enabled or disabled, Rook fails over all monitors, configuring them to enable or disable host networking.
- `spec.network.Provider` : When updated from being empty to "host", Rook fails over all monitors, configuring them to enable or disable host networking.
- `spec.network.multiClusterService`: When enabled or disabled, Rook fails over all monitors, configuring them to start (or stop) using service IPs compatible with the multi-cluster service.
@@ -2232,7 +2232,7 @@ spec:
description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6
type: boolean
hostNetwork:
description: HostNetwork to enable host network
description: HostNetwork to enable host network. If host networking is enabled or disabled on a running cluster, then the operator will automatically fail over all the mons to apply the new network settings.
type: boolean
ipFamily:
description: IPFamily is the single stack IPv6 or IPv4 protocol
@@ -2252,7 +2252,7 @@ spec:
type: boolean
type: object
provider:
description: Provider is what provides network connectivity to the cluster e.g. "host" or "multus"
description: Provider is what provides network connectivity to the cluster e.g. "host" or "multus". If the Provider is updated from being empty to "host" on a running cluster, then the operator will automatically fail over all the mons to apply the "host" network settings.
enum:
- ""
- host
+2 -2
View File
@@ -2230,7 +2230,7 @@ spec:
description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6
type: boolean
hostNetwork:
description: HostNetwork to enable host network
description: HostNetwork to enable host network. If host networking is enabled or disabled on a running cluster, then the operator will automatically fail over all the mons to apply the new network settings.
type: boolean
ipFamily:
description: IPFamily is the single stack IPv6 or IPv4 protocol
@@ -2250,7 +2250,7 @@ spec:
type: boolean
type: object
provider:
description: Provider is what provides network connectivity to the cluster e.g. "host" or "multus"
description: Provider is what provides network connectivity to the cluster e.g. "host" or "multus". If the Provider is updated from being empty to "host" on a running cluster, then the operator will automatically fail over all the mons to apply the "host" network settings.
enum:
- ""
- host
+5 -2
View File
@@ -2317,7 +2317,8 @@ type SSSDSidecarAdditionalFile struct {
// NetworkSpec for Ceph includes backward compatibility code
// +kubebuilder:validation:XValidation:message="at least one network selector must be specified when using multus",rule="!has(self.provider) || (self.provider != 'multus' || (self.provider == 'multus' && size(self.selectors) > 0))"
type NetworkSpec struct {
// Provider is what provides network connectivity to the cluster e.g. "host" or "multus"
// Provider is what provides network connectivity to the cluster e.g. "host" or "multus".
// If the Provider is updated from being empty to "host" on a running cluster, then the operator will automatically fail over all the mons to apply the "host" network settings.
// +kubebuilder:validation:XValidation:message="network provider must be disabled (reverted to empty string) before a new provider is enabled",rule="self == '' || self == oldSelf"
// +nullable
// +optional
@@ -2363,7 +2364,9 @@ type NetworkSpec struct {
// +optional
Connections *ConnectionsSpec `json:"connections,omitempty"`
// HostNetwork to enable host network
// HostNetwork to enable host network.
// If host networking is enabled or disabled on a running cluster, then the operator will automatically fail over all the mons to
// apply the new network settings.
// +optional
HostNetwork bool `json:"hostNetwork,omitempty"`
+19
View File
@@ -1343,6 +1343,12 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
return nil
}
// skip update if mon fail over is required due to change in hostnetwork settings
if isMonIPUpdateRequiredForHostNetwork(m.DaemonName, m.UseHostNetwork, &c.spec.Network) {
c.monsToFailover.Insert(m.DaemonName)
return nil
}
// the existing deployment may have a node selector. if the cluster
// isn't using host networking and the deployment is using pvc storage,
// then the node selector can be removed. this may happen after
@@ -1408,6 +1414,19 @@ func (c *Cluster) startMon(m *monConfig, schedule *controller.MonScheduleInfo) e
return nil
}
func isMonIPUpdateRequiredForHostNetwork(mon string, isMonUsingHostNetwork bool, network *cephv1.NetworkSpec) bool {
isHostNetworkEnabledInSpec := network.IsHost()
if isHostNetworkEnabledInSpec && !isMonUsingHostNetwork {
logger.Infof("host network is enabled for the cluster but mon %q is not running on host IP address", mon)
return true
} else if !isHostNetworkEnabledInSpec && isMonUsingHostNetwork {
logger.Infof("host network is disabled for the cluster but mon %q is still running on host IP address", mon)
return true
}
return false
}
func hasMonPathChanged(d *apps.Deployment, claim *v1.PersistentVolumeClaim) bool {
if d.Labels["pvc_name"] == "" && claim != nil {
logger.Infof("skipping update for mon %q where path has changed from hostPath to pvc", d.Name)
+40 -1
View File
@@ -44,6 +44,7 @@ import (
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
)
// generate a standard mon config from a mon id w/ default port and IP 2.4.6.{1,2,3,...}
@@ -120,7 +121,8 @@ func newCluster(context *clusterd.Context, namespace string, allowMultiplePerNod
mapping: &opcontroller.Mapping{
Schedule: map[string]*opcontroller.MonScheduleInfo{},
},
ownerInfo: ownerInfo,
ownerInfo: ownerInfo,
monsToFailover: sets.New[string](),
}
}
@@ -917,3 +919,40 @@ func TestHasMonPathChanged(t *testing.T) {
assert.False(t, hasMonPathChanged(monDeployment, nil))
})
}
func TestIsMonIPUpdateRequiredForHostNetwork(t *testing.T) {
t.Run("both cluster and mon are set to use host network", func(t *testing.T) {
hostNetwork := &cephv1.NetworkSpec{HostNetwork: true}
monUsingHostNetwork := true
assert.False(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
})
t.Run("both cluster and mon are not set for host network", func(t *testing.T) {
hostNetwork := &cephv1.NetworkSpec{}
monUsingHostNetwork := false
assert.False(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
})
t.Run("cluster is set for host networking but mon pod is not", func(t *testing.T) {
hostNetwork := &cephv1.NetworkSpec{HostNetwork: true}
monUsingHostNetwork := false
assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
})
t.Run("mon is using host networking but cluster is updated to not use host network ", func(t *testing.T) {
hostNetwork := &cephv1.NetworkSpec{}
monUsingHostNetwork := true
assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
})
t.Run("mon is using host networking and cluster is set host network via NetworkProviderHost ", func(t *testing.T) {
hostNetwork := &cephv1.NetworkSpec{Provider: cephv1.NetworkProviderHost}
monUsingHostNetwork := true
assert.False(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
})
t.Run("mon is not using host networking but cluster is updated to use host network via NetworkProviderHost ", func(t *testing.T) {
hostNetwork := &cephv1.NetworkSpec{Provider: cephv1.NetworkProviderHost}
monUsingHostNetwork := false
assert.True(t, isMonIPUpdateRequiredForHostNetwork("a", monUsingHostNetwork, hostNetwork))
})
}