Merge pull request #13890 from BlaineEXE/csi-disable-holder
csi: allow force disabling holder pods
This commit is contained in:
14 files changed
+530
-32
No files matched your search
@@ -1533,6 +1533,9 @@ jobs:
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
run: tests/scripts/github-action-helper.sh build_rook
|
||||
|
||||
- name: allow holder pod deployment
|
||||
run: sed -i "s|CSI_DISABLE_HOLDER_PODS|# CSI_DISABLE_HOLDER_PODS|g" "deploy/examples/operator.yaml"
|
||||
|
||||
- name: validate-yaml
|
||||
run: tests/scripts/github-action-helper.sh validate_yaml
|
||||
|
||||
@@ -1598,6 +1601,9 @@ jobs:
|
||||
- name: setup cluster resources
|
||||
uses: ./.github/workflows/canary-test-config
|
||||
|
||||
- name: allow holder pod deployment
|
||||
run: sed -i "s|CSI_DISABLE_HOLDER_PODS|# CSI_DISABLE_HOLDER_PODS|g" "deploy/examples/operator.yaml"
|
||||
|
||||
- name: set Ceph version in CephCluster manifest
|
||||
run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}"
|
||||
|
||||
|
||||
@@ -79,6 +79,27 @@ to or from host networking after you update this setting, you will need to
|
||||
[failover the mons](../../Storage-Configuration/Advanced/ceph-mon-health.md#failing-over-a-monitor)
|
||||
in order to have mons on the desired network configuration.
|
||||
|
||||
## CSI Host Networking
|
||||
|
||||
Host networking for CSI pods is controlled independently from CephCluster networking. CSI can be
|
||||
deployed with host networking or pod networking. CSI uses host networking by default, which is the
|
||||
recommended configuration. CSI can be forced to use pod networking by setting the operator config
|
||||
`CSI_ENABLE_HOST_NETWORK: "false"`.
|
||||
|
||||
When CSI uses pod networking (`"false"` value), it is critical that `csi-rbdplugin`,
|
||||
`csi-cephfsplugin`, and `csi-nfsplugin` pods are not deleted or updated without following a special
|
||||
process outlined below. If one of these pods is deleted, it will cause all existing PVCs on the
|
||||
pod's node to hang permanently until all application pods are restarted.
|
||||
|
||||
The process for updating CSI plugin pods is to follow the following steps on each Kubernetes node
|
||||
sequentially:
|
||||
1. `cordon` and `drain` the node
|
||||
2. When the node is drained, delete the plugin pod on the node (optionally, the node can be rebooted)
|
||||
3. `uncordon` the node
|
||||
4. Proceed to the next node when pods on the node rerun and stabilize
|
||||
|
||||
For modifications, see [Modifying CSI Networking](#modifying-csi-networking).
|
||||
|
||||
## Multus
|
||||
`network.provider: multus`
|
||||
|
||||
@@ -91,6 +112,53 @@ While any CNI may be used, the intent is to allow use of CNIs which allow Ceph t
|
||||
specific host interfaces. This improves latency and bandwidth while preserving host-level network
|
||||
isolation.
|
||||
|
||||
### Multus Prerequisites
|
||||
|
||||
These prerequisites apply when:
|
||||
- CephCluster `network.selector['public']` is specified, AND
|
||||
- Operator config `CSI_ENABLE_HOST_NETWORK` is `"true"` (or unspecified), AND
|
||||
- Operator config `CSI_DISABLE_HOLDER_PODS` is `"true"`
|
||||
|
||||
If any of the above do not apply, these prerequisites can be skipped.
|
||||
|
||||
In order for host network-enabled Ceph-CSI to communicate with a Multus-enabled CephCluster, some
|
||||
setup is required for Kubernetes hosts.
|
||||
|
||||
These prerequisites require an understanding of how Multus networks are configured and how Rook uses
|
||||
them. Following sections will help clarify questions that may arise here.
|
||||
|
||||
Two basic requirements must be met:
|
||||
|
||||
1. Kubernetes hosts must be able to route successfully to the Multus public network.
|
||||
2. Pods on the Multus public network must be able to route successfully to Kubernetes hosts.
|
||||
|
||||
These two requirements can be broken down further as follows:
|
||||
|
||||
1. For routing Kubernetes hosts to the Multus public network, each host must ensure the following:
|
||||
1. the host must have an interface connected to the Multus public network (the "public-network-interface").
|
||||
2. the "public-network-interface" must have an IP address.
|
||||
3. a route must exist to direct traffic destined for pods on the Multus public network through
|
||||
the "public-network-interface".
|
||||
2. For routing pods on the Multus public network to Kubernetes hosts, the public
|
||||
NetworkAttachementDefinition must be configured to ensure the following:
|
||||
1. The definition must have its IP Address Management (IPAM) configured to route traffic destined
|
||||
for nodes through the network.
|
||||
3. To ensure routing between the two networks works properly, no IP address assigned to a node can
|
||||
overlap with any IP address assigned to a pod on the Multus public network.
|
||||
|
||||
These requirements require careful planning, but some methods are able to meet these requirements
|
||||
more easily than others. [Examples are provided after the full document](#multus-examples) to help
|
||||
understand and implement these requirements.
|
||||
|
||||
!!! Tip
|
||||
Keep in mind that there are often ten or more Rook/Ceph pods per host. The pod address space may
|
||||
need to be an order of magnitude larger (or more) than the host address space to allow the
|
||||
storage cluster to grow in the future.
|
||||
|
||||
If these prerequisites are not achievable, the remaining option is to set the Rook operator config
|
||||
`CSI_ENABLE_HOST_NETWORK: "false"` as documented in the [CSI Host Networking](#csi-host-networking)
|
||||
section.
|
||||
|
||||
### Multus Configuration
|
||||
|
||||
Refer to [Multus documentation](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/how-to-use.md)
|
||||
@@ -113,10 +181,9 @@ apiVersion: "k8s.cni.cncf.io/v1"
|
||||
kind: NetworkAttachmentDefinition
|
||||
metadata:
|
||||
name: ceph-multus-net
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
config: '{
|
||||
"cniVersion": "0.3.0",
|
||||
"cniVersion": "0.3.1",
|
||||
"type": "macvlan",
|
||||
"master": "eth0",
|
||||
"mode": "bridge",
|
||||
@@ -127,13 +194,16 @@ spec:
|
||||
}'
|
||||
```
|
||||
|
||||
* Note that the example above does not specify information that would route pods on the network to
|
||||
Kubernetes hosts.
|
||||
* Ensure that `master` matches the network interface on hosts that you want to use.
|
||||
It must be the same across all hosts.
|
||||
* CNI type `macvlan` is highly recommended.
|
||||
* CNI type [macvlan](https://www.cni.dev/plugins/current/main/macvlan/) is highly recommended.
|
||||
It has less CPU and memory overhead compared to traditional Linux `bridge` configurations.
|
||||
* IPAM type `whereabouts` is recommended because it ensures each pod gets an IP address unique
|
||||
within the Kubernetes cluster. No DHCP server is required. If a DHCP server is present on the
|
||||
network, ensure the IP range does not overlap with the DHCP server's range.
|
||||
* IPAM type [whereabouts](https://github.com/k8snetworkplumbingwg/whereabouts) is recommended
|
||||
because it ensures each pod gets an IP address unique within the Kubernetes cluster. No DHCP
|
||||
server is required. If a DHCP server is present on the network, ensure the IP range does not
|
||||
overlap with the DHCP server's range.
|
||||
|
||||
NetworkAttachmentDefinitions are selected for the desired Ceph network using `selectors`. Selector
|
||||
values should include the namespace in which the NAD is present. `public` and `cluster` may be
|
||||
@@ -189,3 +259,333 @@ NAD is attached to the container, allowing the daemon to communicate with the re
|
||||
There is work in progress to fix this issue in the
|
||||
[multus-service](https://github.com/k8snetworkplumbingwg/multus-service) repository. At the time of
|
||||
writing it's unclear when this will be supported.
|
||||
|
||||
### Multus examples
|
||||
|
||||
#### Macvlan, Whereabouts, Node Dynamic IPs
|
||||
|
||||
The network plan for this cluster will be as follows:
|
||||
- The underlying network supporting the public network will be attached to hosts at `eth0`
|
||||
- Macvlan will be used to attach pods to `eth0`
|
||||
- Pods and nodes will have separate IP ranges
|
||||
- Nodes will get the IP range 192.168.252.0/22 (this allows up to 1024 hosts)
|
||||
- Nodes will have IPs assigned dynamically via DHCP
|
||||
(DHCP configuration is outside the scope of this document)
|
||||
- Pods will get the IP range 192.168.0.0/18 (this allows up to 16,384 Rook/Ceph pods)
|
||||
- Whereabouts will be used to assign IPs to the Multus public network
|
||||
|
||||
Node configuration must allow nodes to route to pods on the Multus public network.
|
||||
|
||||
Because pods will be connecting via Macvlan, and because Macvlan does not allow hosts and pods to
|
||||
route between each other, the host must also be connected via Macvlan.
|
||||
|
||||
Because the host IP range is different from the pod IP range, a route must be added to include the
|
||||
pod range.
|
||||
|
||||
Such a configuration should be equivalent to the following:
|
||||
|
||||
```console
|
||||
ip link add public-shim link eth0 type macvlan mode bridge
|
||||
ip link set public-shim up
|
||||
dhclient public-shim # gets IP in range 192.168.252.0/22
|
||||
ip route add 192.168.0.0/18 dev public-shim
|
||||
```
|
||||
|
||||
The NetworkAttachmentDefinition for the public network would look like the following, using
|
||||
Whereabouts' `exclude` option to simplify the `range` request.
|
||||
|
||||
The Whereabouts `routes[].dst` option
|
||||
([is not well documented](https://gist.github.com/dougbtv/b41c759e9b9aee6a3fe210f09da8e835))
|
||||
but allows adding routing pods to hosts via the Multus public network.
|
||||
|
||||
```yaml
|
||||
apiVersion: "k8s.cni.cncf.io/v1"
|
||||
kind: NetworkAttachmentDefinition
|
||||
metadata:
|
||||
name: public-net
|
||||
# namespace: rook-ceph # (optional) operator namespace
|
||||
spec:
|
||||
config: '{
|
||||
"cniVersion": "0.3.1",
|
||||
"type": "macvlan",
|
||||
"master": "eth0",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "whereabouts",
|
||||
"range": "1192.168.0.0/18",
|
||||
"routes": [
|
||||
{"dst": "192.168.252.0/22"}
|
||||
]
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
#### Macvlan, Whereabouts, Node Static IPs
|
||||
|
||||
The network plan for this cluster will be as follows:
|
||||
- The underlying network supporting the public network will be attached to hosts at `eth0`
|
||||
- Macvlan will be used to attach pods to `eth0`
|
||||
- Pods and nodes will share the IP range 192.168.0.0/16
|
||||
- Nodes will get the IP range 192.168.252.0/22 (this allows up to 1024 hosts)
|
||||
- Pods will get the remainder of the ranges (192.168.0.1 to 192.168.251.255)
|
||||
- Whereabouts will be used to assign IPs to the Multus public network
|
||||
- Nodes will have IPs assigned statically via PXE configs
|
||||
(PXE configuration and static IP management is outside the scope of this document)
|
||||
|
||||
PXE configuration for the nodes must apply a configuration to nodes to allow nodes to route to pods
|
||||
on the Multus public network.
|
||||
|
||||
Because pods will be connecting via Macvlan, and because Macvlan does not allow hosts and pods to
|
||||
route between each other, the host must also be connected via Macvlan.
|
||||
|
||||
Because the host IP range is a subset of the whole range, a route must be added to include the whole
|
||||
range.
|
||||
|
||||
Such a configuration should be equivalent to the following:
|
||||
|
||||
```console
|
||||
ip link add public-shim link eth0 type macvlan mode bridge
|
||||
ip addr add 192.168.252.${STATIC_IP}/22 dev public-shim
|
||||
ip link set public-shim up
|
||||
ip route add 192.168.0.0/16 dev public-shim
|
||||
```
|
||||
|
||||
The NetworkAttachmentDefinition for the public network would look like the following, using
|
||||
Whereabouts' `exclude` option to simplify the `range` request. The Whereabouts `routes[].dst` option
|
||||
ensures pods route to hosts via the Multus public network.
|
||||
|
||||
```yaml
|
||||
apiVersion: "k8s.cni.cncf.io/v1"
|
||||
kind: NetworkAttachmentDefinition
|
||||
metadata:
|
||||
name: public-net
|
||||
spec:
|
||||
config: '{
|
||||
"cniVersion": "0.3.1",
|
||||
"type": "macvlan",
|
||||
"master": "eth0",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "whereabouts",
|
||||
"range": "192.168.0.0/16",
|
||||
"exclude": [
|
||||
"192.168.252.0/22"
|
||||
],
|
||||
"routes": [
|
||||
{"dst": "192.168.252.0/22"}
|
||||
]
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
#### Macvlan, DHCP
|
||||
|
||||
The network plan for this cluster will be as follows:
|
||||
- The underlying network supporting the public network will be attached to hosts at `eth0`
|
||||
- Macvlan will be used to attach pods to `eth0`
|
||||
- Pods and nodes will share the IP range 192.168.0.0/16
|
||||
- DHCP will be used to ensure nodes and pods get unique IP addresses
|
||||
|
||||
Node configuration must allow nodes to route to pods on the Multus public network.
|
||||
|
||||
Because pods will be connecting via Macvlan, and because Macvlan does not allow hosts and pods to
|
||||
route between each other, the host must also be connected via Macvlan.
|
||||
|
||||
Such a configuration should be equivalent to the following:
|
||||
|
||||
```console
|
||||
ip link add public-shim link eth0 type macvlan mode bridge
|
||||
ip link set public-shim up
|
||||
dhclient public-shim # gets IP in range 192.168.0.0/16
|
||||
```
|
||||
|
||||
The NetworkAttachmentDefinition for the public network would look like the following.
|
||||
|
||||
```yaml
|
||||
apiVersion: "k8s.cni.cncf.io/v1"
|
||||
kind: NetworkAttachmentDefinition
|
||||
metadata:
|
||||
name: public-net
|
||||
spec:
|
||||
config: '{
|
||||
"cniVersion": "0.3.1",
|
||||
"type": "macvlan",
|
||||
"master": "eth0",
|
||||
"mode": "bridge",
|
||||
"ipam": {
|
||||
"type": "dhcp",
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
## Modifying CSI networking
|
||||
|
||||
### Disabling Holder Pods with Multus and CSI Host Networking
|
||||
|
||||
This migration section applies in the following scenarios:
|
||||
- CephCluster `network.provider` is `"multus"`, AND
|
||||
- Operator config `CSI_DISABLE_HOLDER_PODS` is changed to `"true"`, AND
|
||||
- Operator config `CSI_ENABLE_HOST_NETWORK` is (or is modified to be) `"true"`
|
||||
|
||||
If the scenario does not apply, skip ahead to the
|
||||
[Disabling Holder Pods](#disabling-holder-pods) section below.
|
||||
|
||||
**Step 1**
|
||||
Before setting `CSI_ENABLE_HOST_NETWORK: "true"` and `CSI_DISABLE_HOLDER_PODS: "true"`, thoroughly
|
||||
read through the [Multus Prerequisites section](#multus-prerequisites). Use the prerequisites
|
||||
section to develop a plan for modifying host configurations as well as the public
|
||||
NetworkAttachmentDefinition.
|
||||
|
||||
Once the plan is developed, execute the plan by following the steps below.
|
||||
|
||||
**Step 2**
|
||||
First, modify the public NetworkAttachmentDefinition as needed. For example, it may be necessary to
|
||||
add the `routes` directive to the Whereabouts IPAM configuration as in
|
||||
[this example](#macvlan-whereabouts-node-static-ips).
|
||||
|
||||
**Step 3**
|
||||
Next, modify the host configurations in the host configuration system. The host configuration system
|
||||
may be something like PXE, ignition config, cloud-init, Ansible, or any other such system. A node
|
||||
reboot is likely necessary to apply configuration updates, but wait until the next step to reboot
|
||||
nodes.
|
||||
|
||||
**Step 4**
|
||||
After the NetworkAttachmentDefinition is modified, OSD pods must be restarted. It is easiest to
|
||||
complete this requirement at the same time nodes are being rebooted to apply configuration updates.
|
||||
|
||||
For each node in the Kubernetes cluster:
|
||||
1. `cordon` and `drain` the node
|
||||
2. Wait for all pods to drain
|
||||
3. Reboot the node, ensuring the new host configuration will be applied
|
||||
4. `uncordon` and `undrain` the node
|
||||
5. Wait for the node to be rehydrated and stable
|
||||
6. Proceed to the next node
|
||||
|
||||
By following this process, host configurations will be updated, and OSDs are also automatically
|
||||
restarted as part of the `drain` and `undrain` process on each node.
|
||||
|
||||
OSDs can be restarted manually if node configuration updates do not require reboot.
|
||||
|
||||
**Step 5**
|
||||
Once all nodes are running the new configuration and all OSDs have been restarted, check that the
|
||||
new node and NetworkAttachmentDefinition configurations are compatible. To do so, verify that each
|
||||
node can `ping` OSD pods via the public network.
|
||||
|
||||
Use the [toolbox](../../Troubleshooting/ceph-toolbox.md) or the
|
||||
[kubectl plugin](../../Troubleshooting/kubectl-plugin.md) to list OSD IPs.
|
||||
|
||||
The example below uses
|
||||
the kubectl plugin, and the OSD public network is 192.168.20.0/24.
|
||||
```console
|
||||
$ kubectl rook-ceph ceph osd dump | grep 'osd\.'
|
||||
osd.0 up in weight 1 up_from 7 up_thru 0 down_at 0 last_clean_interval [0,0) [v2:192.168.20.19:6800/213587265,v1:192.168.20.19:6801/213587265] [v2:192.168.30.1:6800/213587265,v1:192.168.30.1:6801/213587265] exists,up 7ebbc19a-d45a-4b12-8fef-0f9423a59e78
|
||||
osd.1 up in weight 1 up_from 24 up_thru 24 down_at 20 last_clean_interval [8,23) [v2:192.168.20.20:6800/3144257456,v1:192.168.20.20:6801/3144257456] [v2:192.168.30.2:6804/3145257456,v1:192.168.30.2:6805/3145257456] exists,up 146b27da-d605-4138-9748-65603ed0dfa5
|
||||
osd.2 up in weight 1 up_from 21 up_thru 0 down_at 20 last_clean_interval [18,20) [v2:192.168.20.21:6800/1809748134,v1:192.168.20.21:6801/1809748134] [v2:192.168.30.3:6804/1810748134,v1:192.168.30.3:6805/1810748134] exists,up ff3d6592-634e-46fd-a0e4-4fe9fafc0386
|
||||
```
|
||||
|
||||
Now check that each node (NODE) can reach OSDs over the public network:
|
||||
```console
|
||||
$ ssh user@NODE
|
||||
$ user@NODE $> ping -c3 192.168.20.19
|
||||
# [truncated, successful output]
|
||||
$ user@NODE $> ping -c3 192.168.20.20
|
||||
# [truncated, successful output]
|
||||
$ user@NODE $> ping -c3 192.168.20.21
|
||||
# [truncated, successful output]
|
||||
```
|
||||
|
||||
If any node does not get a successful `ping` to a running OSD, it is not safe to proceed. A problem
|
||||
may arise here for many reasons. Some reasons include: the host may not be properly attached to the
|
||||
Multus public network, the public NetworkAttachmentDefinition may not be properly configured to
|
||||
route back to the host, the host may have a firewall rule blocking the connection in either
|
||||
direction, or the network switch may have a firewall rule blocking the connection. Diagnose and fix
|
||||
the issue, then return to **Step 1**.
|
||||
|
||||
**Step 6**
|
||||
If the above check succeeds for all nodes, proceed with the
|
||||
[Disabling Holder Pods](#disabling-holder-pods) steps below.
|
||||
|
||||
### Disabling Holder Pods
|
||||
|
||||
This migration section applies when `CSI_DISABLE_HOLDER_PODS` is changed to `"true"`.
|
||||
|
||||
**Step 1**
|
||||
If any CephClusters have Multus enabled (`network.provider: "multus"`), follow the
|
||||
[Disabling Holder Pods with Multus and CSI Host Networking](#disabling-holder-pods-with-multus-and-csi-host-networking)
|
||||
steps above before continuing.
|
||||
|
||||
**Step 2**
|
||||
Begin by setting `CSI_DISABLE_HOLDER_PODS: "true"` (and `CSI_ENABLE_HOST_NETWORK: "true"` if desired).
|
||||
|
||||
After this, `csi-*plugin-*` pods will restart, and `csi-*plugin-holder-*` pods will remain running.
|
||||
|
||||
**Step 3**
|
||||
Check that CSI pods are using the correct host networking configuration using the example below as
|
||||
guidance (in the example, `CSI_ENABLE_HOST_NETWORK` is `"true"`):
|
||||
```console
|
||||
$ kubectl -n rook-ceph get -o yaml daemonsets.apps csi-rbdplugin | grep -i hostnetwork
|
||||
hostNetwork: true
|
||||
$ kubectl -n rook-ceph get -o yaml daemonsets.apps csi-cephfsplugin | grep -i hostnetwork
|
||||
hostNetwork: true
|
||||
$ kubectl -n rook-ceph get -o yaml daemonsets.apps csi-nfsplugin | grep -i hostnetwork
|
||||
hostNetwork: true
|
||||
```
|
||||
|
||||
**Step 4**
|
||||
At this stage, PVCs for running applications are still using the holder pods. These PVCs must be
|
||||
migrated from the holder to the new network. Follow the below process to do so.
|
||||
|
||||
For each node in the Kubernetes cluster:
|
||||
1. `cordon` and `drain` the node
|
||||
2. Wait for all pods to drain
|
||||
3. Delete all `csi-*plugin-holder*` pods on the node (a new holder will take it's place)
|
||||
4. `uncordon` and `undrain` the node
|
||||
5. Wait for the node to be rehydrated and stable
|
||||
6. Proceed to the next node
|
||||
|
||||
**Step 5**
|
||||
After this process is done for all Kubernetes nodes, it is safe to delete the `csi-*plugin-holder*`
|
||||
daemonsets.
|
||||
|
||||
Delete the holder daemonsets using the example below as guidance:
|
||||
```console
|
||||
$ kubectl -n rook-ceph get daemonset -o name | grep plugin-holder
|
||||
daemonset.apps/csi-cephfsplugin-holder-my-cluster
|
||||
daemonset.apps/csi-rbdplugin-holder-my-cluster
|
||||
|
||||
$ kubectl -n rook-ceph delete daemonset.apps/csi-cephfsplugin-holder-my-cluster
|
||||
daemonset.apps "csi-cephfsplugin-holder-my-cluster" deleted
|
||||
|
||||
$ kubectl -n rook-ceph delete daemonset.apps/csi-rbdplugin-holder-my-cluster
|
||||
daemonset.apps "csi-rbdplugin-holder-my-cluster" deleted
|
||||
```
|
||||
|
||||
**Step 6**
|
||||
The migration is now complete! Congratulations!
|
||||
|
||||
### Applying CSI Networking
|
||||
|
||||
This migration section applies in the following scenario:
|
||||
- `CSI_ENABLE_HOST_NETWORK` is modified, AND
|
||||
- `CSI_DISABLE_HOLDER_PODS` is `"true"`
|
||||
|
||||
**Step 1**
|
||||
If `CSI_DISABLE_HOLDER_PODS` is unspecified or is `"false"`, follow the
|
||||
[Disabling Holder Pods](#disabling-holder-pods) section first.
|
||||
|
||||
**Step 2**
|
||||
Begin by setting the desired `CSI_ENABLE_HOST_NETWORK` value.
|
||||
|
||||
**Step 3**
|
||||
At this stage, PVCs for running applications are still using the the old network. These PVCs must be
|
||||
migrated to the new network. Follow the below process to do so.
|
||||
|
||||
For each node in the Kubernetes cluster:
|
||||
1. `cordon` and `drain` the node
|
||||
2. Wait for all pods to drain
|
||||
3. `uncordon` and `undrain` the node
|
||||
4. Wait for the node to be rehydrated and stable
|
||||
5. Proceed to the next node
|
||||
|
||||
**Step 4**
|
||||
The migration is now complete! Congratulations!
|
||||
@@ -80,6 +80,7 @@ The following table lists the configurable parameters of the rook-operator chart
|
||||
| `csi.csiRBDPluginVolume` | The volume of the CephCSI RBD plugin DaemonSet | `nil` |
|
||||
| `csi.csiRBDPluginVolumeMount` | The volume mounts of the CephCSI RBD plugin DaemonSet | `nil` |
|
||||
| `csi.csiRBDProvisionerResource` | CEPH CSI RBD provisioner resource requirement list csi-omap-generator resources will be applied only if `enableOMAPGenerator` is set to `true` | see values.yaml |
|
||||
| `csi.disableHolderPods` | Deprecation note: Rook uses "holder" pods to allow CSI to connect to the multus public network without needing hosts to the network. Holder pods are being deprecated. See issue for details: https://github.com/rook/rook/issues/13055. New Rook deployments should set this to "true". | `true` |
|
||||
| `csi.enableCSIEncryption` | Enable Ceph CSI PVC encryption support | `false` |
|
||||
| `csi.enableCSIHostNetwork` | Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary in some network configurations where the SDN does not provide access to an external cluster or there is significant drop in read/write performance | `true` |
|
||||
| `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver | `true` |
|
||||
|
||||
@@ -256,3 +256,12 @@ This cluster is finished:
|
||||
At this point, the Rook operator should be running version `rook/ceph:v1.13.0`.
|
||||
|
||||
Verify the CephCluster health using the [health verification doc](health-verification.md).
|
||||
|
||||
### **6. Disable CSI holder pods**
|
||||
CSI "holder" pods are frequently reported objects of confusion and struggle in Rook. Because of
|
||||
this, they are being deprecated and will be removed in Rook v1.16.
|
||||
|
||||
If there are any CephClusters that use the non-default network setting `network.provider: "multus"`,
|
||||
or if the operator config `CSI_ENABLE_HOST_NETWORK` is set to `"false"`, perform migration steps to
|
||||
remove holder pods by setting `CSI_REMOVE_HOLDER_PODS: "true"` after following this migration guide:
|
||||
[Modifying CSI Networking](./../CRDs/Cluster/network-providers.md#modifying-csi-networking)
|
||||
@@ -7,6 +7,10 @@ read affinity setting in cephCluster CR (CSIDriverOptions section) in [PR](https
|
||||
- updating `netNamespaceFilePath` for all clusterIDs in rook-ceph-csi-config configMap in [PR](https://github.com/rook/rook/pull/13613)
|
||||
- Issue: The netNamespaceFilePath isn't updated in the CSI config map for all the clusterIDs when `CSI_ENABLE_HOST_NETWORK` is set to false in `operator.yaml`
|
||||
- Impact: This results in the unintended network configurations, with pods using the host networking instead of pod networking.
|
||||
- Rook is beginning the process of deprecating holder pods. This is especially important for Multus
|
||||
users. Helm chart users should take care to set the new config `disableHolderPods: false` if they
|
||||
are using Multus and still using the `rook-ceph` chart's default `values.yaml`. Special upgrade
|
||||
docs for multus can be found [here](Documentation/CRDs/Cluster/network-providers.md#migrating-to-remove-multus-holder-pods).
|
||||
|
||||
## Features
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ data:
|
||||
CSI_ENABLE_ENCRYPTION: {{ .Values.csi.enableCSIEncryption | quote }}
|
||||
CSI_ENABLE_OMAP_GENERATOR: {{ .Values.csi.enableOMAPGenerator | quote }}
|
||||
CSI_ENABLE_HOST_NETWORK: {{ .Values.csi.enableCSIHostNetwork | quote }}
|
||||
CSI_DISABLE_HOLDER_PODS: {{ .Values.csi.disableHolderPods | quote }}
|
||||
CSI_ENABLE_METADATA: {{ .Values.csi.enableMetadata | quote }}
|
||||
CSI_ENABLE_VOLUME_GROUP_SNAPSHOT: {{ .Values.csi.enableVolumeGroupSnapshot | quote }}
|
||||
{{- if .Values.csi.csiDriverNamePrefix }}
|
||||
|
||||
@@ -85,6 +85,10 @@ csi:
|
||||
# in some network configurations where the SDN does not provide access to an external cluster or
|
||||
# there is significant drop in read/write performance
|
||||
enableCSIHostNetwork: true
|
||||
# -- Deprecation note: Rook uses "holder" pods to allow CSI to connect to the multus public network
|
||||
# without needing hosts to the network. Holder pods are being deprecated. See issue for details:
|
||||
# https://github.com/rook/rook/issues/13055. New Rook deployments should set this to "true".
|
||||
disableHolderPods: true
|
||||
# -- Enable Snapshotter in CephFS provisioner pod
|
||||
enableCephfsSnapshotter: true
|
||||
# -- Enable Snapshotter in NFS provisioner pod
|
||||
|
||||
@@ -137,6 +137,11 @@ data:
|
||||
# there is significant drop in read/write performance.
|
||||
# CSI_ENABLE_HOST_NETWORK: "true"
|
||||
|
||||
# Deprecation note: Rook uses "holder" pods to allow CSI to connect to the multus public network
|
||||
# without needing hosts to the network. Holder pods are being deprecated. See issue for details:
|
||||
# https://github.com/rook/rook/issues/13055. New Rook deployments should set this to "true".
|
||||
CSI_DISABLE_HOLDER_PODS: "true"
|
||||
|
||||
# Set logging level for cephCSI containers maintained by the cephCSI.
|
||||
# Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
|
||||
# CSI_LOG_LEVEL: "0"
|
||||
|
||||
@@ -44,6 +44,11 @@ data:
|
||||
# there is significant drop in read/write performance.
|
||||
# CSI_ENABLE_HOST_NETWORK: "true"
|
||||
|
||||
# Deprecation note: Rook uses "holder" pods to allow CSI to connect to the multus public network
|
||||
# without needing hosts to the network. Holder pods are being deprecated. See issue for details:
|
||||
# https://github.com/rook/rook/issues/13055. New Rook deployments should set this to "true".
|
||||
CSI_DISABLE_HOLDER_PODS: "true"
|
||||
|
||||
# Set to true to enable adding volume metadata on the CephFS subvolume and RBD images.
|
||||
# Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images.
|
||||
# Hence enable metadata is false by default.
|
||||
|
||||
@@ -341,6 +341,7 @@ func updateCSIDriverOptions(curr, clusterKey string,
|
||||
}
|
||||
}
|
||||
|
||||
updateNetNamespaceFilePath(clusterKey, cc)
|
||||
return formatCsiClusterConfig(cc)
|
||||
}
|
||||
|
||||
|
||||
@@ -503,6 +503,8 @@ func TestUpdateCSIDriverOptions(t *testing.T) {
|
||||
clusterKey string
|
||||
csiDriverOptions *cephv1.CSIDriverSpec
|
||||
}
|
||||
holderEnabled = true
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
|
||||
@@ -130,6 +130,9 @@ func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Requ
|
||||
return reconcileResponse, err
|
||||
}
|
||||
|
||||
// allow overriding for unit tests
|
||||
var reconcileSaveCSIDriverOptions = SaveCSIDriverOptions
|
||||
|
||||
func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, error) {
|
||||
// reconcileResult is used to communicate the result of the reconciliation back to the caller
|
||||
var reconcileResult reconcile.Result
|
||||
@@ -192,6 +195,17 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
|
||||
return reconcile.Result{}, errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'")
|
||||
}
|
||||
|
||||
csiDisableHolders, err := strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_DISABLE_HOLDER_PODS", "false"))
|
||||
if err != nil {
|
||||
return reconcile.Result{}, errors.Wrap(err, "failed to parse value for 'CSI_DISABLE_HOLDER_PODS'")
|
||||
}
|
||||
|
||||
// begin each reconcile with the assumption that holder pods won't be deployed
|
||||
// the loop below will determine with each reconcile if they need deployed
|
||||
// without this, holder pods won't be removed unless the operator is restarted
|
||||
r.clustersWithHolder = []ClusterDetail{}
|
||||
holderEnabled = false
|
||||
|
||||
for i, cluster := range cephClusters.Items {
|
||||
if !cluster.DeletionTimestamp.IsZero() {
|
||||
logger.Debugf("ceph cluster %q is being deleting, no need to reconcile the csi driver", request.NamespacedName)
|
||||
@@ -203,28 +217,40 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
holderEnabled := !csiHostNetworkEnabled || cluster.Spec.Network.IsMultus()
|
||||
// Load cluster info for later use in updating the ceph-csi configmap
|
||||
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec)
|
||||
if err != nil {
|
||||
// This avoids a requeue with exponential backoff and allows the controller to reconcile
|
||||
// more quickly when the cluster is ready.
|
||||
if errors.Is(err, opcontroller.ClusterInfoNoClusterNoSecret) {
|
||||
logger.Infof("cluster info for cluster %q is not ready yet, will retry in %s, proceeding with ready clusters", cluster.Name, opcontroller.WaitForRequeueIfCephClusterNotReady.RequeueAfter.String())
|
||||
reconcileResult = opcontroller.WaitForRequeueIfCephClusterNotReady
|
||||
continue
|
||||
}
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to load cluster info for cluster %q", cluster.Name)
|
||||
}
|
||||
clusterInfo.OwnerInfo = k8sutil.NewOwnerInfo(&cephClusters.Items[i], r.scheme)
|
||||
|
||||
// is holder enabled for this cluster?
|
||||
thisHolderEnabled := (!csiHostNetworkEnabled || cluster.Spec.Network.IsMultus()) && !csiDisableHolders
|
||||
|
||||
// Do we have a multus cluster or csi host network disabled?
|
||||
// If so deploy the plugin holder with the fsid attached
|
||||
if holderEnabled {
|
||||
// Load cluster info for later use in updating the ceph-csi configmap
|
||||
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec)
|
||||
if err != nil {
|
||||
// This avoids a requeue with exponential backoff and allows the controller to reconcile
|
||||
// more quickly when the cluster is ready.
|
||||
if errors.Is(err, opcontroller.ClusterInfoNoClusterNoSecret) {
|
||||
logger.Infof("cluster info for cluster %q is not ready yet, will retry in %s, proceeding with ready clusters", cluster.Name, opcontroller.WaitForRequeueIfCephClusterNotReady.RequeueAfter.String())
|
||||
reconcileResult = opcontroller.WaitForRequeueIfCephClusterNotReady
|
||||
continue
|
||||
}
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to load cluster info for cluster %q", cluster.Name)
|
||||
}
|
||||
clusterInfo.OwnerInfo = k8sutil.NewOwnerInfo(&cephClusters.Items[i], r.scheme)
|
||||
logger.Debugf("cluster %q is running on multus or CSI_ENABLE_HOST_NETWORK is false, deploying the ceph-csi plugin holder", cluster.Name)
|
||||
|
||||
if thisHolderEnabled {
|
||||
logger.Debugf("cluster %q: deploying the ceph-csi plugin holder", cluster.Name)
|
||||
r.clustersWithHolder = append(r.clustersWithHolder, ClusterDetail{cluster: &cephClusters.Items[i], clusterInfo: clusterInfo})
|
||||
|
||||
// holder pods are enabled globally if any cluster needs a holder pod
|
||||
holderEnabled = true
|
||||
} else {
|
||||
logger.Debugf("not a multus cluster %q or CSI_ENABLE_HOST_NETWORK is true, not deploying the ceph-csi plugin holder", request.NamespacedName)
|
||||
logger.Debugf("cluster %q: not deploying the ceph-csi plugin holder", request.NamespacedName)
|
||||
}
|
||||
|
||||
// if holder pods were disabled, the controller needs to update the configmap for each
|
||||
// cephcluster to remove the net namespace file path
|
||||
err = reconcileSaveCSIDriverOptions(r.context.Clientset, cluster.Namespace, clusterInfo)
|
||||
if err != nil {
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to update CSI driver options for cluster %q", cluster.Name)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
|
||||
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
@@ -35,11 +36,20 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
)
|
||||
|
||||
func TestCephCSIController(t *testing.T) {
|
||||
oldReconcileSaveCSIDriverOptions := reconcileSaveCSIDriverOptions
|
||||
defer func() { reconcileSaveCSIDriverOptions = oldReconcileSaveCSIDriverOptions }()
|
||||
saveCSIDriverOptionsCalledForClusterNS := []string{}
|
||||
reconcileSaveCSIDriverOptions = func(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *client.ClusterInfo) error {
|
||||
saveCSIDriverOptionsCalledForClusterNS = append(saveCSIDriverOptionsCalledForClusterNS, clusterNamespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx := context.TODO()
|
||||
var (
|
||||
name = "rook-ceph"
|
||||
@@ -129,8 +139,25 @@ func TestCephCSIController(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
// Mock clusterInfo
|
||||
secrets := map[string][]byte{
|
||||
"fsid": []byte(name),
|
||||
"mon-secret": []byte("monsecret"),
|
||||
"admin-secret": []byte("adminsecret"),
|
||||
}
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-ceph-mon",
|
||||
Namespace: namespace,
|
||||
},
|
||||
Data: secrets,
|
||||
Type: k8sutil.RookType,
|
||||
}
|
||||
_, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
s := scheme.Scheme
|
||||
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &v1.ConfigMap{})
|
||||
saveCSIDriverOptionsCalledForClusterNS = []string{}
|
||||
|
||||
object := []runtime.Object{
|
||||
cephCluster,
|
||||
@@ -156,6 +183,8 @@ func TestCephCSIController(t *testing.T) {
|
||||
ds, err := c.Clientset.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, len(ds.Items), ds)
|
||||
|
||||
assert.Equal(t, []string{namespace}, saveCSIDriverOptionsCalledForClusterNS)
|
||||
})
|
||||
|
||||
t.Run("success ceph csi deployment with multus", func(t *testing.T) {
|
||||
@@ -209,6 +238,7 @@ func TestCephCSIController(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
s := scheme.Scheme
|
||||
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &v1.ConfigMap{})
|
||||
saveCSIDriverOptionsCalledForClusterNS = []string{}
|
||||
|
||||
object := []runtime.Object{
|
||||
cephCluster,
|
||||
@@ -240,5 +270,7 @@ func TestCephCSIController(t *testing.T) {
|
||||
assert.Equal(t, "csi-rbdplugin", ds.Items[2].Name)
|
||||
assert.Equal(t, "csi-rbdplugin-holder-rook-ceph", ds.Items[3].Name)
|
||||
assert.Equal(t, `[{"name":"public-net","namespace":"rook-ceph"}]`, ds.Items[1].Spec.Template.Annotations["k8s.v1.cni.cncf.io/networks"], ds.Items[1].Spec.Template.Annotations)
|
||||
|
||||
assert.Equal(t, []string{namespace}, saveCSIDriverOptionsCalledForClusterNS)
|
||||
})
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -419,14 +420,6 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
|
||||
})
|
||||
}
|
||||
|
||||
holderEnabled = !CSIParam.EnableCSIHostNetwork
|
||||
|
||||
for i := range r.clustersWithHolder {
|
||||
if r.clustersWithHolder[i].cluster.Spec.Network.IsMultus() {
|
||||
holderEnabled = true
|
||||
break
|
||||
}
|
||||
}
|
||||
// get common provisioner tolerations and node affinity
|
||||
provisionerTolerations := getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{})
|
||||
provisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{})
|
||||
@@ -946,6 +939,15 @@ func GenerateNetNamespaceFilePath(ctx context.Context, client client.Client, clu
|
||||
return "", errors.Wrap(err, "failed to get operator's configmap")
|
||||
}
|
||||
|
||||
// net namespace file path is empty string if holder pods are disabled
|
||||
csiDisableHolders, err := strconv.ParseBool(k8sutil.GetValue(opConfig.Data, "CSI_DISABLE_HOLDER_PODS", "false"))
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to parse value for 'CSI_DISABLE_HOLDER_PODS'")
|
||||
}
|
||||
if csiDisableHolders {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
switch driverName {
|
||||
case RBDDriverShortName:
|
||||
driverSuffix = rbdDriverSuffix
|
||||
|
||||
Reference in new issue
Block a user