From 0e115c44bf5e5c0ecff0cc650ddc33fc05703df4 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 14 Feb 2024 10:46:00 +0100 Subject: [PATCH] core: fix cephfs pvc network fencing The cephfs PVC might exist on the kubernetes node object but due to some timing issues the ip might not be visible on the ceph cluster or the client might already got evicted or disconnected from ceph cluster. In this case we will not be able to get IP details for the subvolume and we dont have any check for empty ip's in the code and rook tries to create NetworkFence CR with empty Ip's and the NetworkFence will get moved to the Failed state. This PR adds the necessary check and logging to prevent this one. Signed-off-by: Madhu Rajanna --- pkg/operator/ceph/cluster/watcher.go | 7 ++++++- pkg/operator/ceph/cluster/watcher_test.go | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index 6b8f08d40..f9162b11c 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -438,10 +438,15 @@ func (c *clientCluster) fenceCephFSVolume( return fmt.Errorf("failed to list watchers for cephfs pool/subvoumeName %s/%s. %v", cephFSPV.Spec.CSI.VolumeAttributes["pool"], cephFSPV.Spec.CSI.VolumeAttributes["subvolumeName"], err) } ips, err := cephFSMDSClientMarshal(buf, cephFSPV) - if err != nil || ips == nil { + if err != nil { return fmt.Errorf("failed to unmarshal cephfs mds output. %v", err) } + if len(ips) == 0 { + logger.Infof("no active mds clients found for cephfs volume %q", cephFSPV.Name) + return nil + } + err = c.createNetworkFence(ctx, cephFSPV, node, cluster, ips, cephfsDriver) if err != nil { return fmt.Errorf("failed to create network fence for node %q. %v", node.Name, err) diff --git a/pkg/operator/ceph/cluster/watcher_test.go b/pkg/operator/ceph/cluster/watcher_test.go index 6e527b910..29b6d60f8 100644 --- a/pkg/operator/ceph/cluster/watcher_test.go +++ b/pkg/operator/ceph/cluster/watcher_test.go @@ -179,7 +179,7 @@ func TestHandleNodeFailure(t *testing.T) { case command == "ceph" && args[0] == "status": return `{"entity":[{"addr": [{"addr": "10.244.0.12:0", "nonce":3247243972}]}], "client_metadata":{"root":"/"}}`, nil case command == "ceph" && args[0] == "tell": - return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/"}}]`, nil + return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca"}}]`, nil } return "", errors.Errorf("unexpected rbd/ceph command %q", args) @@ -250,6 +250,7 @@ func TestHandleNodeFailure(t *testing.T) { VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", VolumeAttributes: map[string]string{ "fsName": "myfs", + "subvolumePath": "/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", "subvolumeName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", }, },