From 4f555dbbcbdc3242d85d6fa1703054237544ee36 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Mon, 4 Mar 2024 11:08:11 -0700 Subject: [PATCH] csi: allow force disabling holder pods Add new CSI_DISABLE_HOLDER_PODS option for rook-ceph-operator. This option will disable holder pods when set to "true". In the long term, Rook plans to deprecate the holder pods entirely. This new option will allow users to choose to migrate their clusters to non-holder clusters when they are ready and able, giving them time to gracefully migrate before the holders are permanently removed. This option is set to "false" by default so that upgrading users don't have their CSI pods modified unexpectedly. Example manifests are modified to set this value to true so that new clusters will not deploy holder pods. Migrating users are provided with documentation to instruct them about the new requirements they need to satisfy to successfully remove holder pods, a procedure for migrating pods from holder to non-holder mounts, and a way to delete holder pods once they are no longer in use. When users set CSI_DISABLE_HOLDER_PODS="true", the CSI controller will no longer deploy or update the holder pod Daemonsets, but it does not delete any existing Daemonsets. This allows already-attached PVCs to continue operating normally with their network connection continuing to exist in the current holder pod. This is critical to avoid causing ia cluster-wide storage outage. More info: https://github.com/rook/rook/issues/13055 Signed-off-by: Blaine Gardner --- .github/workflows/canary-integration-test.yml | 6 + .../CRDs/Cluster/network-providers.md | 412 +++++++++++++++++- Documentation/Helm-Charts/operator-chart.md | 1 + Documentation/Upgrade/rook-upgrade.md | 9 + PendingReleaseNotes.md | 4 + .../charts/rook-ceph/templates/configmap.yaml | 1 + deploy/charts/rook-ceph/values.yaml | 4 + deploy/examples/operator-openshift.yaml | 5 + deploy/examples/operator.yaml | 5 + pkg/operator/ceph/csi/cluster_config.go | 1 + pkg/operator/ceph/csi/cluster_config_test.go | 2 + pkg/operator/ceph/csi/controller.go | 62 ++- pkg/operator/ceph/csi/controller_test.go | 32 ++ pkg/operator/ceph/csi/spec.go | 18 +- 14 files changed, 530 insertions(+), 32 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index d11bcb6a4..5df527162 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -1533,6 +1533,9 @@ jobs: shell: bash --noprofile --norc -eo pipefail -x {0} run: tests/scripts/github-action-helper.sh build_rook + - name: allow holder pod deployment + run: sed -i "s|CSI_DISABLE_HOLDER_PODS|# CSI_DISABLE_HOLDER_PODS|g" "deploy/examples/operator.yaml" + - name: validate-yaml run: tests/scripts/github-action-helper.sh validate_yaml @@ -1598,6 +1601,9 @@ jobs: - name: setup cluster resources uses: ./.github/workflows/canary-test-config + - name: allow holder pod deployment + run: sed -i "s|CSI_DISABLE_HOLDER_PODS|# CSI_DISABLE_HOLDER_PODS|g" "deploy/examples/operator.yaml" + - name: set Ceph version in CephCluster manifest run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" diff --git a/Documentation/CRDs/Cluster/network-providers.md b/Documentation/CRDs/Cluster/network-providers.md index a74e6c7a0..f98f44eea 100644 --- a/Documentation/CRDs/Cluster/network-providers.md +++ b/Documentation/CRDs/Cluster/network-providers.md @@ -79,6 +79,27 @@ to or from host networking after you update this setting, you will need to [failover the mons](../../Storage-Configuration/Advanced/ceph-mon-health.md#failing-over-a-monitor) in order to have mons on the desired network configuration. +## CSI Host Networking + +Host networking for CSI pods is controlled independently from CephCluster networking. CSI can be +deployed with host networking or pod networking. CSI uses host networking by default, which is the +recommended configuration. CSI can be forced to use pod networking by setting the operator config +`CSI_ENABLE_HOST_NETWORK: "false"`. + +When CSI uses pod networking (`"false"` value), it is critical that `csi-rbdplugin`, +`csi-cephfsplugin`, and `csi-nfsplugin` pods are not deleted or updated without following a special +process outlined below. If one of these pods is deleted, it will cause all existing PVCs on the +pod's node to hang permanently until all application pods are restarted. + +The process for updating CSI plugin pods is to follow the following steps on each Kubernetes node +sequentially: +1. `cordon` and `drain` the node +2. When the node is drained, delete the plugin pod on the node (optionally, the node can be rebooted) +3. `uncordon` the node +4. Proceed to the next node when pods on the node rerun and stabilize + +For modifications, see [Modifying CSI Networking](#modifying-csi-networking). + ## Multus `network.provider: multus` @@ -91,6 +112,53 @@ While any CNI may be used, the intent is to allow use of CNIs which allow Ceph t specific host interfaces. This improves latency and bandwidth while preserving host-level network isolation. +### Multus Prerequisites + +These prerequisites apply when: +- CephCluster `network.selector['public']` is specified, AND +- Operator config `CSI_ENABLE_HOST_NETWORK` is `"true"` (or unspecified), AND +- Operator config `CSI_DISABLE_HOLDER_PODS` is `"true"` + +If any of the above do not apply, these prerequisites can be skipped. + +In order for host network-enabled Ceph-CSI to communicate with a Multus-enabled CephCluster, some +setup is required for Kubernetes hosts. + +These prerequisites require an understanding of how Multus networks are configured and how Rook uses +them. Following sections will help clarify questions that may arise here. + +Two basic requirements must be met: + +1. Kubernetes hosts must be able to route successfully to the Multus public network. +2. Pods on the Multus public network must be able to route successfully to Kubernetes hosts. + +These two requirements can be broken down further as follows: + +1. For routing Kubernetes hosts to the Multus public network, each host must ensure the following: + 1. the host must have an interface connected to the Multus public network (the "public-network-interface"). + 2. the "public-network-interface" must have an IP address. + 3. a route must exist to direct traffic destined for pods on the Multus public network through + the "public-network-interface". +2. For routing pods on the Multus public network to Kubernetes hosts, the public + NetworkAttachementDefinition must be configured to ensure the following: + 1. The definition must have its IP Address Management (IPAM) configured to route traffic destined + for nodes through the network. +3. To ensure routing between the two networks works properly, no IP address assigned to a node can + overlap with any IP address assigned to a pod on the Multus public network. + +These requirements require careful planning, but some methods are able to meet these requirements +more easily than others. [Examples are provided after the full document](#multus-examples) to help +understand and implement these requirements. + +!!! Tip + Keep in mind that there are often ten or more Rook/Ceph pods per host. The pod address space may + need to be an order of magnitude larger (or more) than the host address space to allow the + storage cluster to grow in the future. + +If these prerequisites are not achievable, the remaining option is to set the Rook operator config +`CSI_ENABLE_HOST_NETWORK: "false"` as documented in the [CSI Host Networking](#csi-host-networking) +section. + ### Multus Configuration Refer to [Multus documentation](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/how-to-use.md) @@ -113,10 +181,9 @@ apiVersion: "k8s.cni.cncf.io/v1" kind: NetworkAttachmentDefinition metadata: name: ceph-multus-net - namespace: rook-ceph spec: config: '{ - "cniVersion": "0.3.0", + "cniVersion": "0.3.1", "type": "macvlan", "master": "eth0", "mode": "bridge", @@ -127,13 +194,16 @@ spec: }' ``` +* Note that the example above does not specify information that would route pods on the network to + Kubernetes hosts. * Ensure that `master` matches the network interface on hosts that you want to use. It must be the same across all hosts. -* CNI type `macvlan` is highly recommended. +* CNI type [macvlan](https://www.cni.dev/plugins/current/main/macvlan/) is highly recommended. It has less CPU and memory overhead compared to traditional Linux `bridge` configurations. -* IPAM type `whereabouts` is recommended because it ensures each pod gets an IP address unique - within the Kubernetes cluster. No DHCP server is required. If a DHCP server is present on the - network, ensure the IP range does not overlap with the DHCP server's range. +* IPAM type [whereabouts](https://github.com/k8snetworkplumbingwg/whereabouts) is recommended + because it ensures each pod gets an IP address unique within the Kubernetes cluster. No DHCP + server is required. If a DHCP server is present on the network, ensure the IP range does not + overlap with the DHCP server's range. NetworkAttachmentDefinitions are selected for the desired Ceph network using `selectors`. Selector values should include the namespace in which the NAD is present. `public` and `cluster` may be @@ -189,3 +259,333 @@ NAD is attached to the container, allowing the daemon to communicate with the re There is work in progress to fix this issue in the [multus-service](https://github.com/k8snetworkplumbingwg/multus-service) repository. At the time of writing it's unclear when this will be supported. + +### Multus examples + +#### Macvlan, Whereabouts, Node Dynamic IPs + +The network plan for this cluster will be as follows: +- The underlying network supporting the public network will be attached to hosts at `eth0` +- Macvlan will be used to attach pods to `eth0` +- Pods and nodes will have separate IP ranges +- Nodes will get the IP range 192.168.252.0/22 (this allows up to 1024 hosts) +- Nodes will have IPs assigned dynamically via DHCP + (DHCP configuration is outside the scope of this document) +- Pods will get the IP range 192.168.0.0/18 (this allows up to 16,384 Rook/Ceph pods) +- Whereabouts will be used to assign IPs to the Multus public network + +Node configuration must allow nodes to route to pods on the Multus public network. + +Because pods will be connecting via Macvlan, and because Macvlan does not allow hosts and pods to +route between each other, the host must also be connected via Macvlan. + +Because the host IP range is different from the pod IP range, a route must be added to include the +pod range. + +Such a configuration should be equivalent to the following: + +```console +ip link add public-shim link eth0 type macvlan mode bridge +ip link set public-shim up +dhclient public-shim # gets IP in range 192.168.252.0/22 +ip route add 192.168.0.0/18 dev public-shim +``` + +The NetworkAttachmentDefinition for the public network would look like the following, using +Whereabouts' `exclude` option to simplify the `range` request. + +The Whereabouts `routes[].dst` option +([is not well documented](https://gist.github.com/dougbtv/b41c759e9b9aee6a3fe210f09da8e835)) +but allows adding routing pods to hosts via the Multus public network. + +```yaml +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: public-net + # namespace: rook-ceph # (optional) operator namespace +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "range": "1192.168.0.0/18", + "routes": [ + {"dst": "192.168.252.0/22"} + ] + } + }' +``` + +#### Macvlan, Whereabouts, Node Static IPs + +The network plan for this cluster will be as follows: +- The underlying network supporting the public network will be attached to hosts at `eth0` +- Macvlan will be used to attach pods to `eth0` +- Pods and nodes will share the IP range 192.168.0.0/16 +- Nodes will get the IP range 192.168.252.0/22 (this allows up to 1024 hosts) +- Pods will get the remainder of the ranges (192.168.0.1 to 192.168.251.255) +- Whereabouts will be used to assign IPs to the Multus public network +- Nodes will have IPs assigned statically via PXE configs + (PXE configuration and static IP management is outside the scope of this document) + +PXE configuration for the nodes must apply a configuration to nodes to allow nodes to route to pods +on the Multus public network. + +Because pods will be connecting via Macvlan, and because Macvlan does not allow hosts and pods to +route between each other, the host must also be connected via Macvlan. + +Because the host IP range is a subset of the whole range, a route must be added to include the whole +range. + +Such a configuration should be equivalent to the following: + +```console +ip link add public-shim link eth0 type macvlan mode bridge +ip addr add 192.168.252.${STATIC_IP}/22 dev public-shim +ip link set public-shim up +ip route add 192.168.0.0/16 dev public-shim +``` + +The NetworkAttachmentDefinition for the public network would look like the following, using +Whereabouts' `exclude` option to simplify the `range` request. The Whereabouts `routes[].dst` option +ensures pods route to hosts via the Multus public network. + +```yaml +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: public-net +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "range": "192.168.0.0/16", + "exclude": [ + "192.168.252.0/22" + ], + "routes": [ + {"dst": "192.168.252.0/22"} + ] + } + }' +``` + +#### Macvlan, DHCP + +The network plan for this cluster will be as follows: +- The underlying network supporting the public network will be attached to hosts at `eth0` +- Macvlan will be used to attach pods to `eth0` +- Pods and nodes will share the IP range 192.168.0.0/16 +- DHCP will be used to ensure nodes and pods get unique IP addresses + +Node configuration must allow nodes to route to pods on the Multus public network. + +Because pods will be connecting via Macvlan, and because Macvlan does not allow hosts and pods to +route between each other, the host must also be connected via Macvlan. + +Such a configuration should be equivalent to the following: + +```console +ip link add public-shim link eth0 type macvlan mode bridge +ip link set public-shim up +dhclient public-shim # gets IP in range 192.168.0.0/16 +``` + +The NetworkAttachmentDefinition for the public network would look like the following. + +```yaml +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: public-net +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "dhcp", + } + }' +``` + +## Modifying CSI networking + +### Disabling Holder Pods with Multus and CSI Host Networking + +This migration section applies in the following scenarios: +- CephCluster `network.provider` is `"multus"`, AND +- Operator config `CSI_DISABLE_HOLDER_PODS` is changed to `"true"`, AND +- Operator config `CSI_ENABLE_HOST_NETWORK` is (or is modified to be) `"true"` + +If the scenario does not apply, skip ahead to the +[Disabling Holder Pods](#disabling-holder-pods) section below. + +**Step 1** +Before setting `CSI_ENABLE_HOST_NETWORK: "true"` and `CSI_DISABLE_HOLDER_PODS: "true"`, thoroughly +read through the [Multus Prerequisites section](#multus-prerequisites). Use the prerequisites +section to develop a plan for modifying host configurations as well as the public +NetworkAttachmentDefinition. + +Once the plan is developed, execute the plan by following the steps below. + +**Step 2** +First, modify the public NetworkAttachmentDefinition as needed. For example, it may be necessary to +add the `routes` directive to the Whereabouts IPAM configuration as in +[this example](#macvlan-whereabouts-node-static-ips). + +**Step 3** +Next, modify the host configurations in the host configuration system. The host configuration system +may be something like PXE, ignition config, cloud-init, Ansible, or any other such system. A node +reboot is likely necessary to apply configuration updates, but wait until the next step to reboot +nodes. + +**Step 4** +After the NetworkAttachmentDefinition is modified, OSD pods must be restarted. It is easiest to +complete this requirement at the same time nodes are being rebooted to apply configuration updates. + +For each node in the Kubernetes cluster: +1. `cordon` and `drain` the node +2. Wait for all pods to drain +3. Reboot the node, ensuring the new host configuration will be applied +4. `uncordon` and `undrain` the node +5. Wait for the node to be rehydrated and stable +6. Proceed to the next node + +By following this process, host configurations will be updated, and OSDs are also automatically +restarted as part of the `drain` and `undrain` process on each node. + +OSDs can be restarted manually if node configuration updates do not require reboot. + +**Step 5** +Once all nodes are running the new configuration and all OSDs have been restarted, check that the +new node and NetworkAttachmentDefinition configurations are compatible. To do so, verify that each +node can `ping` OSD pods via the public network. + +Use the [toolbox](../../Troubleshooting/ceph-toolbox.md) or the +[kubectl plugin](../../Troubleshooting/kubectl-plugin.md) to list OSD IPs. + +The example below uses +the kubectl plugin, and the OSD public network is 192.168.20.0/24. +```console +$ kubectl rook-ceph ceph osd dump | grep 'osd\.' +osd.0 up in weight 1 up_from 7 up_thru 0 down_at 0 last_clean_interval [0,0) [v2:192.168.20.19:6800/213587265,v1:192.168.20.19:6801/213587265] [v2:192.168.30.1:6800/213587265,v1:192.168.30.1:6801/213587265] exists,up 7ebbc19a-d45a-4b12-8fef-0f9423a59e78 +osd.1 up in weight 1 up_from 24 up_thru 24 down_at 20 last_clean_interval [8,23) [v2:192.168.20.20:6800/3144257456,v1:192.168.20.20:6801/3144257456] [v2:192.168.30.2:6804/3145257456,v1:192.168.30.2:6805/3145257456] exists,up 146b27da-d605-4138-9748-65603ed0dfa5 +osd.2 up in weight 1 up_from 21 up_thru 0 down_at 20 last_clean_interval [18,20) [v2:192.168.20.21:6800/1809748134,v1:192.168.20.21:6801/1809748134] [v2:192.168.30.3:6804/1810748134,v1:192.168.30.3:6805/1810748134] exists,up ff3d6592-634e-46fd-a0e4-4fe9fafc0386 +``` + +Now check that each node (NODE) can reach OSDs over the public network: +```console +$ ssh user@NODE +$ user@NODE $> ping -c3 192.168.20.19 +# [truncated, successful output] +$ user@NODE $> ping -c3 192.168.20.20 +# [truncated, successful output] +$ user@NODE $> ping -c3 192.168.20.21 +# [truncated, successful output] +``` + +If any node does not get a successful `ping` to a running OSD, it is not safe to proceed. A problem +may arise here for many reasons. Some reasons include: the host may not be properly attached to the +Multus public network, the public NetworkAttachmentDefinition may not be properly configured to +route back to the host, the host may have a firewall rule blocking the connection in either +direction, or the network switch may have a firewall rule blocking the connection. Diagnose and fix +the issue, then return to **Step 1**. + +**Step 6** +If the above check succeeds for all nodes, proceed with the +[Disabling Holder Pods](#disabling-holder-pods) steps below. + +### Disabling Holder Pods + +This migration section applies when `CSI_DISABLE_HOLDER_PODS` is changed to `"true"`. + +**Step 1** +If any CephClusters have Multus enabled (`network.provider: "multus"`), follow the +[Disabling Holder Pods with Multus and CSI Host Networking](#disabling-holder-pods-with-multus-and-csi-host-networking) +steps above before continuing. + +**Step 2** +Begin by setting `CSI_DISABLE_HOLDER_PODS: "true"` (and `CSI_ENABLE_HOST_NETWORK: "true"` if desired). + +After this, `csi-*plugin-*` pods will restart, and `csi-*plugin-holder-*` pods will remain running. + +**Step 3** +Check that CSI pods are using the correct host networking configuration using the example below as +guidance (in the example, `CSI_ENABLE_HOST_NETWORK` is `"true"`): +```console +$ kubectl -n rook-ceph get -o yaml daemonsets.apps csi-rbdplugin | grep -i hostnetwork + hostNetwork: true +$ kubectl -n rook-ceph get -o yaml daemonsets.apps csi-cephfsplugin | grep -i hostnetwork + hostNetwork: true +$ kubectl -n rook-ceph get -o yaml daemonsets.apps csi-nfsplugin | grep -i hostnetwork + hostNetwork: true +``` + +**Step 4** +At this stage, PVCs for running applications are still using the holder pods. These PVCs must be +migrated from the holder to the new network. Follow the below process to do so. + +For each node in the Kubernetes cluster: +1. `cordon` and `drain` the node +2. Wait for all pods to drain +3. Delete all `csi-*plugin-holder*` pods on the node (a new holder will take it's place) +4. `uncordon` and `undrain` the node +5. Wait for the node to be rehydrated and stable +6. Proceed to the next node + +**Step 5** +After this process is done for all Kubernetes nodes, it is safe to delete the `csi-*plugin-holder*` +daemonsets. + +Delete the holder daemonsets using the example below as guidance: +```console +$ kubectl -n rook-ceph get daemonset -o name | grep plugin-holder +daemonset.apps/csi-cephfsplugin-holder-my-cluster +daemonset.apps/csi-rbdplugin-holder-my-cluster + +$ kubectl -n rook-ceph delete daemonset.apps/csi-cephfsplugin-holder-my-cluster +daemonset.apps "csi-cephfsplugin-holder-my-cluster" deleted + +$ kubectl -n rook-ceph delete daemonset.apps/csi-rbdplugin-holder-my-cluster +daemonset.apps "csi-rbdplugin-holder-my-cluster" deleted +``` + +**Step 6** +The migration is now complete! Congratulations! + +### Applying CSI Networking + +This migration section applies in the following scenario: +- `CSI_ENABLE_HOST_NETWORK` is modified, AND +- `CSI_DISABLE_HOLDER_PODS` is `"true"` + +**Step 1** +If `CSI_DISABLE_HOLDER_PODS` is unspecified or is `"false"`, follow the +[Disabling Holder Pods](#disabling-holder-pods) section first. + +**Step 2** +Begin by setting the desired `CSI_ENABLE_HOST_NETWORK` value. + +**Step 3** +At this stage, PVCs for running applications are still using the the old network. These PVCs must be +migrated to the new network. Follow the below process to do so. + +For each node in the Kubernetes cluster: +1. `cordon` and `drain` the node +2. Wait for all pods to drain +3. `uncordon` and `undrain` the node +4. Wait for the node to be rehydrated and stable +5. Proceed to the next node + +**Step 4** +The migration is now complete! Congratulations! diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 352c5498e..ad2a706ce 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -80,6 +80,7 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.csiRBDPluginVolume` | The volume of the CephCSI RBD plugin DaemonSet | `nil` | | `csi.csiRBDPluginVolumeMount` | The volume mounts of the CephCSI RBD plugin DaemonSet | `nil` | | `csi.csiRBDProvisionerResource` | CEPH CSI RBD provisioner resource requirement list csi-omap-generator resources will be applied only if `enableOMAPGenerator` is set to `true` | see values.yaml | +| `csi.disableHolderPods` | Deprecation note: Rook uses "holder" pods to allow CSI to connect to the multus public network without needing hosts to the network. Holder pods are being deprecated. See issue for details: https://github.com/rook/rook/issues/13055. New Rook deployments should set this to "true". | `true` | | `csi.enableCSIEncryption` | Enable Ceph CSI PVC encryption support | `false` | | `csi.enableCSIHostNetwork` | Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary in some network configurations where the SDN does not provide access to an external cluster or there is significant drop in read/write performance | `true` | | `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver | `true` | diff --git a/Documentation/Upgrade/rook-upgrade.md b/Documentation/Upgrade/rook-upgrade.md index 5ef74e162..ad1b4aba6 100644 --- a/Documentation/Upgrade/rook-upgrade.md +++ b/Documentation/Upgrade/rook-upgrade.md @@ -256,3 +256,12 @@ This cluster is finished: At this point, the Rook operator should be running version `rook/ceph:v1.13.0`. Verify the CephCluster health using the [health verification doc](health-verification.md). + +### **6. Disable CSI holder pods** +CSI "holder" pods are frequently reported objects of confusion and struggle in Rook. Because of +this, they are being deprecated and will be removed in Rook v1.16. + +If there are any CephClusters that use the non-default network setting `network.provider: "multus"`, +or if the operator config `CSI_ENABLE_HOST_NETWORK` is set to `"false"`, perform migration steps to +remove holder pods by setting `CSI_REMOVE_HOLDER_PODS: "true"` after following this migration guide: +[Modifying CSI Networking](./../CRDs/Cluster/network-providers.md#modifying-csi-networking) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 838182e0c..d7ce4118f 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -7,6 +7,10 @@ read affinity setting in cephCluster CR (CSIDriverOptions section) in [PR](https - updating `netNamespaceFilePath` for all clusterIDs in rook-ceph-csi-config configMap in [PR](https://github.com/rook/rook/pull/13613) - Issue: The netNamespaceFilePath isn't updated in the CSI config map for all the clusterIDs when `CSI_ENABLE_HOST_NETWORK` is set to false in `operator.yaml` - Impact: This results in the unintended network configurations, with pods using the host networking instead of pod networking. +- Rook is beginning the process of deprecating holder pods. This is especially important for Multus + users. Helm chart users should take care to set the new config `disableHolderPods: false` if they + are using Multus and still using the `rook-ceph` chart's default `values.yaml`. Special upgrade + docs for multus can be found [here](Documentation/CRDs/Cluster/network-providers.md#migrating-to-remove-multus-holder-pods). ## Features diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 4ce7b75dc..154e5f340 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -24,6 +24,7 @@ data: CSI_ENABLE_ENCRYPTION: {{ .Values.csi.enableCSIEncryption | quote }} CSI_ENABLE_OMAP_GENERATOR: {{ .Values.csi.enableOMAPGenerator | quote }} CSI_ENABLE_HOST_NETWORK: {{ .Values.csi.enableCSIHostNetwork | quote }} + CSI_DISABLE_HOLDER_PODS: {{ .Values.csi.disableHolderPods | quote }} CSI_ENABLE_METADATA: {{ .Values.csi.enableMetadata | quote }} CSI_ENABLE_VOLUME_GROUP_SNAPSHOT: {{ .Values.csi.enableVolumeGroupSnapshot | quote }} {{- if .Values.csi.csiDriverNamePrefix }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index c01b4016d..6af5017a8 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -85,6 +85,10 @@ csi: # in some network configurations where the SDN does not provide access to an external cluster or # there is significant drop in read/write performance enableCSIHostNetwork: true + # -- Deprecation note: Rook uses "holder" pods to allow CSI to connect to the multus public network + # without needing hosts to the network. Holder pods are being deprecated. See issue for details: + # https://github.com/rook/rook/issues/13055. New Rook deployments should set this to "true". + disableHolderPods: true # -- Enable Snapshotter in CephFS provisioner pod enableCephfsSnapshotter: true # -- Enable Snapshotter in NFS provisioner pod diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index d269206da..2c1506fa2 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -137,6 +137,11 @@ data: # there is significant drop in read/write performance. # CSI_ENABLE_HOST_NETWORK: "true" + # Deprecation note: Rook uses "holder" pods to allow CSI to connect to the multus public network + # without needing hosts to the network. Holder pods are being deprecated. See issue for details: + # https://github.com/rook/rook/issues/13055. New Rook deployments should set this to "true". + CSI_DISABLE_HOLDER_PODS: "true" + # Set logging level for cephCSI containers maintained by the cephCSI. # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. # CSI_LOG_LEVEL: "0" diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 7bb91ac1f..481de81b4 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -44,6 +44,11 @@ data: # there is significant drop in read/write performance. # CSI_ENABLE_HOST_NETWORK: "true" + # Deprecation note: Rook uses "holder" pods to allow CSI to connect to the multus public network + # without needing hosts to the network. Holder pods are being deprecated. See issue for details: + # https://github.com/rook/rook/issues/13055. New Rook deployments should set this to "true". + CSI_DISABLE_HOLDER_PODS: "true" + # Set to true to enable adding volume metadata on the CephFS subvolume and RBD images. # Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. # Hence enable metadata is false by default. diff --git a/pkg/operator/ceph/csi/cluster_config.go b/pkg/operator/ceph/csi/cluster_config.go index 6b9a9ee59..132ca5350 100644 --- a/pkg/operator/ceph/csi/cluster_config.go +++ b/pkg/operator/ceph/csi/cluster_config.go @@ -341,6 +341,7 @@ func updateCSIDriverOptions(curr, clusterKey string, } } + updateNetNamespaceFilePath(clusterKey, cc) return formatCsiClusterConfig(cc) } diff --git a/pkg/operator/ceph/csi/cluster_config_test.go b/pkg/operator/ceph/csi/cluster_config_test.go index 3698e0fb1..fefc036ba 100644 --- a/pkg/operator/ceph/csi/cluster_config_test.go +++ b/pkg/operator/ceph/csi/cluster_config_test.go @@ -503,6 +503,8 @@ func TestUpdateCSIDriverOptions(t *testing.T) { clusterKey string csiDriverOptions *cephv1.CSIDriverSpec } + holderEnabled = true + tests := []struct { name string args args diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index e380d9ea4..1c56183c2 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -130,6 +130,9 @@ func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Requ return reconcileResponse, err } +// allow overriding for unit tests +var reconcileSaveCSIDriverOptions = SaveCSIDriverOptions + func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, error) { // reconcileResult is used to communicate the result of the reconciliation back to the caller var reconcileResult reconcile.Result @@ -192,6 +195,17 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return reconcile.Result{}, errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'") } + csiDisableHolders, err := strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_DISABLE_HOLDER_PODS", "false")) + if err != nil { + return reconcile.Result{}, errors.Wrap(err, "failed to parse value for 'CSI_DISABLE_HOLDER_PODS'") + } + + // begin each reconcile with the assumption that holder pods won't be deployed + // the loop below will determine with each reconcile if they need deployed + // without this, holder pods won't be removed unless the operator is restarted + r.clustersWithHolder = []ClusterDetail{} + holderEnabled = false + for i, cluster := range cephClusters.Items { if !cluster.DeletionTimestamp.IsZero() { logger.Debugf("ceph cluster %q is being deleting, no need to reconcile the csi driver", request.NamespacedName) @@ -203,28 +217,40 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return reconcile.Result{}, nil } - holderEnabled := !csiHostNetworkEnabled || cluster.Spec.Network.IsMultus() + // Load cluster info for later use in updating the ceph-csi configmap + clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec) + if err != nil { + // This avoids a requeue with exponential backoff and allows the controller to reconcile + // more quickly when the cluster is ready. + if errors.Is(err, opcontroller.ClusterInfoNoClusterNoSecret) { + logger.Infof("cluster info for cluster %q is not ready yet, will retry in %s, proceeding with ready clusters", cluster.Name, opcontroller.WaitForRequeueIfCephClusterNotReady.RequeueAfter.String()) + reconcileResult = opcontroller.WaitForRequeueIfCephClusterNotReady + continue + } + return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to load cluster info for cluster %q", cluster.Name) + } + clusterInfo.OwnerInfo = k8sutil.NewOwnerInfo(&cephClusters.Items[i], r.scheme) + + // is holder enabled for this cluster? + thisHolderEnabled := (!csiHostNetworkEnabled || cluster.Spec.Network.IsMultus()) && !csiDisableHolders + // Do we have a multus cluster or csi host network disabled? // If so deploy the plugin holder with the fsid attached - if holderEnabled { - // Load cluster info for later use in updating the ceph-csi configmap - clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec) - if err != nil { - // This avoids a requeue with exponential backoff and allows the controller to reconcile - // more quickly when the cluster is ready. - if errors.Is(err, opcontroller.ClusterInfoNoClusterNoSecret) { - logger.Infof("cluster info for cluster %q is not ready yet, will retry in %s, proceeding with ready clusters", cluster.Name, opcontroller.WaitForRequeueIfCephClusterNotReady.RequeueAfter.String()) - reconcileResult = opcontroller.WaitForRequeueIfCephClusterNotReady - continue - } - return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to load cluster info for cluster %q", cluster.Name) - } - clusterInfo.OwnerInfo = k8sutil.NewOwnerInfo(&cephClusters.Items[i], r.scheme) - logger.Debugf("cluster %q is running on multus or CSI_ENABLE_HOST_NETWORK is false, deploying the ceph-csi plugin holder", cluster.Name) - + if thisHolderEnabled { + logger.Debugf("cluster %q: deploying the ceph-csi plugin holder", cluster.Name) r.clustersWithHolder = append(r.clustersWithHolder, ClusterDetail{cluster: &cephClusters.Items[i], clusterInfo: clusterInfo}) + + // holder pods are enabled globally if any cluster needs a holder pod + holderEnabled = true } else { - logger.Debugf("not a multus cluster %q or CSI_ENABLE_HOST_NETWORK is true, not deploying the ceph-csi plugin holder", request.NamespacedName) + logger.Debugf("cluster %q: not deploying the ceph-csi plugin holder", request.NamespacedName) + } + + // if holder pods were disabled, the controller needs to update the configmap for each + // cephcluster to remove the net namespace file path + err = reconcileSaveCSIDriverOptions(r.context.Clientset, cluster.Namespace, clusterInfo) + if err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to update CSI driver options for cluster %q", cluster.Name) } } diff --git a/pkg/operator/ceph/csi/controller_test.go b/pkg/operator/ceph/csi/controller_test.go index 8f72eb265..2860c6d67 100644 --- a/pkg/operator/ceph/csi/controller_test.go +++ b/pkg/operator/ceph/csi/controller_test.go @@ -26,6 +26,7 @@ import ( rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" @@ -35,11 +36,20 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) func TestCephCSIController(t *testing.T) { + oldReconcileSaveCSIDriverOptions := reconcileSaveCSIDriverOptions + defer func() { reconcileSaveCSIDriverOptions = oldReconcileSaveCSIDriverOptions }() + saveCSIDriverOptionsCalledForClusterNS := []string{} + reconcileSaveCSIDriverOptions = func(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *client.ClusterInfo) error { + saveCSIDriverOptionsCalledForClusterNS = append(saveCSIDriverOptionsCalledForClusterNS, clusterNamespace) + return nil + } + ctx := context.TODO() var ( name = "rook-ceph" @@ -129,8 +139,25 @@ func TestCephCSIController(t *testing.T) { }, }, } + // Mock clusterInfo + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) s := scheme.Scheme s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &v1.ConfigMap{}) + saveCSIDriverOptionsCalledForClusterNS = []string{} object := []runtime.Object{ cephCluster, @@ -156,6 +183,8 @@ func TestCephCSIController(t *testing.T) { ds, err := c.Clientset.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{}) assert.NoError(t, err) assert.Equal(t, 2, len(ds.Items), ds) + + assert.Equal(t, []string{namespace}, saveCSIDriverOptionsCalledForClusterNS) }) t.Run("success ceph csi deployment with multus", func(t *testing.T) { @@ -209,6 +238,7 @@ func TestCephCSIController(t *testing.T) { assert.NoError(t, err) s := scheme.Scheme s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &v1.ConfigMap{}) + saveCSIDriverOptionsCalledForClusterNS = []string{} object := []runtime.Object{ cephCluster, @@ -240,5 +270,7 @@ func TestCephCSIController(t *testing.T) { assert.Equal(t, "csi-rbdplugin", ds.Items[2].Name) assert.Equal(t, "csi-rbdplugin-holder-rook-ceph", ds.Items[3].Name) assert.Equal(t, `[{"name":"public-net","namespace":"rook-ceph"}]`, ds.Items[1].Spec.Template.Annotations["k8s.v1.cni.cncf.io/networks"], ds.Items[1].Spec.Template.Annotations) + + assert.Equal(t, []string{namespace}, saveCSIDriverOptionsCalledForClusterNS) }) } diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 188a9ca85..1a8c5bba8 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -20,6 +20,7 @@ import ( "context" _ "embed" "fmt" + "strconv" "strings" "time" @@ -419,14 +420,6 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI }) } - holderEnabled = !CSIParam.EnableCSIHostNetwork - - for i := range r.clustersWithHolder { - if r.clustersWithHolder[i].cluster.Spec.Network.IsMultus() { - holderEnabled = true - break - } - } // get common provisioner tolerations and node affinity provisionerTolerations := getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{}) provisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}) @@ -946,6 +939,15 @@ func GenerateNetNamespaceFilePath(ctx context.Context, client client.Client, clu return "", errors.Wrap(err, "failed to get operator's configmap") } + // net namespace file path is empty string if holder pods are disabled + csiDisableHolders, err := strconv.ParseBool(k8sutil.GetValue(opConfig.Data, "CSI_DISABLE_HOLDER_PODS", "false")) + if err != nil { + return "", errors.Wrap(err, "failed to parse value for 'CSI_DISABLE_HOLDER_PODS'") + } + if csiDisableHolders { + return "", nil + } + switch driverName { case RBDDriverShortName: driverSuffix = rbdDriverSuffix