From c3f4be43548c317a57ba56c2cabdafcb40daa1aa Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Thu, 22 Feb 2024 12:24:36 +0530 Subject: [PATCH 1/3] build: update csi-addons to v0.8.0 update go.mod to use csi-addons version v0.8.0. Signed-off-by: subhamkrai --- go.mod | 2 +- go.sum | 8 ++++---- pkg/apis/go.mod | 4 ++-- pkg/apis/go.sum | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 03bc15317..b22b9fb54 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/banzaicloud/k8s-objectmatcher v1.8.0 github.com/ceph/go-ceph v0.24.0 github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8 - github.com/csi-addons/kubernetes-csi-addons v0.7.0 + github.com/csi-addons/kubernetes-csi-addons v0.8.0 github.com/gemalto/kmip-go v0.0.10 github.com/go-ini/ini v1.67.0 github.com/google/go-cmp v0.6.0 diff --git a/go.sum b/go.sum index dd6464325..b2c76c4f3 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/csi-addons/kubernetes-csi-addons v0.7.0 h1:Cqon7e9Wnil8gq4/M80uilMJcTv7LypdKJoOVhEb4oM= -github.com/csi-addons/kubernetes-csi-addons v0.7.0/go.mod h1:7thZgf/0YB59yWWv8jb0l6mv1bnPEFh5Kxj4k66pnPs= +github.com/csi-addons/kubernetes-csi-addons v0.8.0 h1:zvYGp4DM6KdQzEX3dQSYKykqJdLZlxpVBJjtpbaqFjs= +github.com/csi-addons/kubernetes-csi-addons v0.8.0/go.mod h1:dvinzoiXlqdOGDpKkYx8Jxl507BzVEEEO+SI0OmBaRI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -613,7 +613,7 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -627,7 +627,7 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4 h1:5RyeLvTSZEn/fDQA6e6+qIvFPssWjreY8pbwfg4/EEQ= github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= diff --git a/pkg/apis/go.mod b/pkg/apis/go.mod index 60ee6704a..c71591266 100644 --- a/pkg/apis/go.mod +++ b/pkg/apis/go.mod @@ -53,8 +53,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.11.0 // indirect - github.com/onsi/gomega v1.27.10 // indirect + github.com/onsi/ginkgo/v2 v2.13.0 // indirect + github.com/onsi/gomega v1.30.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect diff --git a/pkg/apis/go.sum b/pkg/apis/go.sum index 8e617f02e..76cb6a719 100644 --- a/pkg/apis/go.sum +++ b/pkg/apis/go.sum @@ -407,8 +407,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -420,8 +420,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4 h1:5RyeLvTSZEn/fDQA6e6+qIvFPssWjreY8pbwfg4/EEQ= github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= From 5a15f6af302be4d5e21866e22679f30d84d2c621 Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Wed, 21 Feb 2024 16:29:20 +0530 Subject: [PATCH 2/3] csi: check networkFence status before deleting earlier, we were only checking the specs of networkFence to see if unfencing is done before proceeding with deletion but we should also check the status of networkFence before proceeding with deletion. Signed-off-by: subhamkrai (cherry picked from commit 0ace7cf8cf0ffad86fe7d02e0637db09412e436e) --- pkg/operator/ceph/cluster/watcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index 92756e706..aeaadae12 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -604,8 +604,8 @@ func (c *clientCluster) unfenceAndDeleteNetworkFence(ctx context.Context, node c return false, err } - if networkFence.Spec.FenceState != addonsv1alpha1.Unfenced { - logger.Infof("waiting for network fence CR %s to get in %s state before deletion", networkFence.Name, addonsv1alpha1.Unfenced) + if networkFence.Status.Message != addonsv1alpha1.UnFenceOperationSuccessfulMessage { + logger.Infof("waiting for network fence CR %q status to get result %q", networkFence.Name, addonsv1alpha1.UnFenceOperationSuccessfulMessage) return false, err } From 28182e921479e2aad5d4ec9b9b865f3a0c777da5 Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Wed, 21 Feb 2024 16:12:57 +0530 Subject: [PATCH 3/3] csi: remove cephFs networkFence code temporarily cephFs fencing needs more testing and bug fixes to make it in production. We tried multiple fixes but still it need more work. So, removing the cephfs networkFence realted code will rever this commit when we start working on this again. Signed-off-by: subhamkrai (cherry picked from commit bd4250f56e202daf9cf40ae3dd16382cb75663df) --- pkg/operator/ceph/cluster/watcher.go | 171 +--------------------- pkg/operator/ceph/cluster/watcher_test.go | 100 +------------ 2 files changed, 16 insertions(+), 255 deletions(-) diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index aeaadae12..4f899450e 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -60,8 +60,7 @@ var ( // drivers that supports fencing, used in naming networkFence object const ( - rbdDriver = "rbd" - cephfsDriver = "cephfs" + rbdDriver = "rbd" ) func newClientCluster(client client.Client, namespace string, context *clusterd.Context) *clientCluster { @@ -201,11 +200,6 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C return pkgerror.Wrapf(err, "failed to delete rbd network fence for node %q.", node.Name) } - err = c.unfenceAndDeleteNetworkFence(ctx, *node, cluster, cephfsDriver) - if err != nil { - return pkgerror.Wrapf(err, "failed to delete cephFS network fence for node %q.", node.Name) - } - return nil } @@ -217,9 +211,9 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste } logger.Debugf("volumesInuse %s", volumesInuse) - rbdVolumesInUse, cephFSVolumeInUse := getCephVolumesInUse(cluster, volumesInuse) - if len(rbdVolumesInUse) == 0 && len(cephFSVolumeInUse) == 0 { - logger.Debugf("no rbd or cephFS volumes in use for out of service node %q", node.Name) + rbdVolumesInUse := getCephVolumesInUse(cluster, volumesInuse) + if len(rbdVolumesInUse) == 0 { + logger.Debugf("no rbd volumes in use for out of service node %q", node.Name) return nil } @@ -258,46 +252,11 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste } } - if len(cephFSVolumeInUse) != 0 { - cephFSVolumeInUseMap := make(map[string]struct{}) - for _, vol := range cephFSVolumeInUse { - cephFSVolumeInUseMap[vol] = struct{}{} - } - cephFSPVList := listRWOCephFSPV(listPVs, cluster, cephFSVolumeInUseMap) - if len(cephFSPVList) == 0 { - logger.Debug("No cephFS PVs found on the node") - return nil - } - logger.Infof("node %q require fencing, found cephFS volumes in use", node.Name) - clusterInfo, _, _, err := opcontroller.LoadClusterInfo(c.context, ctx, cluster.Namespace, &cluster.Spec) - if err != nil { - return pkgerror.Wrapf(err, "Failed to load cluster info.") - } - - for i := range cephFSPVList { - err = c.fenceCephFSVolume(ctx, node, cluster, clusterInfo, cephFSPVList[i]) - // We only need to create the network fence for any one of cephFS pv. - if err == nil { - break - } - - // continue to fence next rbd volume if active client not found - if stderrors.Is(err, errActiveClientNotFound) { - continue - } - if i == len(cephFSPVList)-1 { - return pkgerror.Wrapf(err, "failed to fence cephFS volumes") - } - logger.Errorf("failed to fence cephFS volumes %q, trying next cephFS volume", cephFSPVList[i].Name) - } - - } - return nil } -func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) ([]string, []string) { - var rbdVolumesInUse, cephFSVolumeInUse []string +func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) []string { + var rbdVolumesInUse []string for _, volume := range volumesInUse { splitVolumeInUseBased := trimeVolumeInUse(volume) @@ -306,13 +265,9 @@ func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.Uniq if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) { rbdVolumesInUse = append(rbdVolumesInUse, splitVolumeInUseBased[1]) } - - if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.cephfs.csi.ceph.com", cluster.Namespace) { - cephFSVolumeInUse = append(cephFSVolumeInUse, splitVolumeInUseBased[1]) - } } - return rbdVolumesInUse, cephFSVolumeInUse + return rbdVolumesInUse } func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string { @@ -351,36 +306,6 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster return listRbdPV } -func listRWOCephFSPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, cephFSVolumesInUse map[string]struct{}) []corev1.PersistentVolume { - var listCephFSPV []corev1.PersistentVolume - - for _, pv := range listPVs.Items { - // Skip if pv is not provisioned by CSI - if pv.Spec.CSI == nil { - logger.Debugf("pv %q is not provisioned by CSI", pv.Name) - continue - } - - if pv.Spec.CSI.Driver == fmt.Sprintf("%s.cephfs.csi.ceph.com", cluster.Namespace) { - // Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes. - if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) { - continue - } - - if pv.Spec.CSI.VolumeAttributes["staticVolume"] == "true" { - logger.Debugf("skipping, static pv %q", pv.Name) - continue - } - // Check if the volume is in use - if _, exists := cephFSVolumesInUse[pv.Spec.CSI.VolumeHandle]; exists { - listCephFSPV = append(listCephFSPV, pv) - } - } - - } - return listCephFSPV -} - // pvSupportsMultiNodeAccess returns true if the PV access modes contain ReadWriteMany or ReadOnlyMany. func pvSupportsMultiNodeAccess(accessModes []corev1.PersistentVolumeAccessMode) bool { for _, accessMode := range accessModes { @@ -423,85 +348,6 @@ func (c *clientCluster) fenceRbdImage( return nil } -func (c *clientCluster) fenceCephFSVolume( - ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster, - clusterInfo *cephclient.ClusterInfo, cephFSPV corev1.PersistentVolume) error { - - logger.Infof("fencing cephfs volume %q on node %q", cephFSPV.Name, node.Name) - - status, err := cephclient.StatusWithUser(c.context, clusterInfo) - if err != nil { - return pkgerror.Wrapf(err, "failed to get ceph status for check active mds") - } - - var activeMDS string - for _, fsRank := range status.Fsmap.ByRank { - if fsRank.Status == "up:active" { - activeMDS = fsRank.Name - } - } - - args := []string{"tell", fmt.Sprintf("mds.%s", activeMDS), "client", "ls", "--format", "json"} - cmd := cephclient.NewCephCommand(c.context, clusterInfo, args) - cmd.JsonOutput = true - - buf, err := cmd.Run() - if err != nil { - return fmt.Errorf("failed to list watchers for cephfs pool/subvoumeName %s/%s. %v", cephFSPV.Spec.CSI.VolumeAttributes["pool"], cephFSPV.Spec.CSI.VolumeAttributes["subvolumeName"], err) - } - ips, err := cephFSMDSClientMarshal(buf, cephFSPV) - if err != nil { - return pkgerror.Wrapf(err, "failed to unmarshal cephfs mds output") - } - - if len(ips) == 0 { - logger.Infof("no active mds clients found for cephfs volume %q", cephFSPV.Name) - return errActiveClientNotFound - } - - err = c.createNetworkFence(ctx, cephFSPV, node, cluster, ips, cephfsDriver) - if err != nil { - return pkgerror.Wrapf(err, "failed to create network fence for node %q", node.Name) - } - - return nil -} - -func cephFSMDSClientMarshal(output []byte, cephFSPV corev1.PersistentVolume) ([]string, error) { - type entity struct { - Addr struct { - Addr string `json:"addr"` - Nonce int `json:"nonce"` - } `json:"addr"` - } - - type clientMetadata struct { - Root string `json:"root"` - } - - type cephFSData struct { - Entity entity `json:"entity"` - ClientMetadata clientMetadata `json:"client_metadata"` - } - - var data []cephFSData - err := json.Unmarshal([]byte(output), &data) - if err != nil { - return []string{}, pkgerror.Wrapf(err, "failed to unmarshal cephFS data output") - } - - watcherIPlist := []string{} - for _, d := range data { - if cephFSPV.Spec.CSI.VolumeAttributes["subvolumePath"] == d.ClientMetadata.Root { - logger.Infof("cephfs mds client ips to fence %v", d.Entity.Addr) - watcherIP := concatenateWatcherIp(d.Entity.Addr.Addr) - watcherIPlist = append(watcherIPlist, watcherIP) - } - } - - return watcherIPlist, nil -} - func rbdStatusUnMarshal(output []byte) ([]string, error) { type rbdStatus struct { Watchers []struct { @@ -524,8 +370,7 @@ func rbdStatusUnMarshal(output []byte) ([]string, error) { } func concatenateWatcherIp(address string) string { - // address is in format `10.63.0.5:0/1254753579` for rbd and - // in the format '10.244.0.12:0' for cephfs + // address is in format `10.63.0.5:0/1254753579` for rbd // split with separation ':0' to remove nounce and concatenating `/32` to define a network with only one IP address watcherIP := strings.Split(address, ":0")[0] + "/32" return watcherIP diff --git a/pkg/operator/ceph/cluster/watcher_test.go b/pkg/operator/ceph/cluster/watcher_test.go index 29b6d60f8..e3585ae81 100644 --- a/pkg/operator/ceph/cluster/watcher_test.go +++ b/pkg/operator/ceph/cluster/watcher_test.go @@ -206,7 +206,6 @@ func TestHandleNodeFailure(t *testing.T) { }, VolumesInUse: []corev1.UniqueVolumeName{ "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", - "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", }, }, } @@ -234,30 +233,6 @@ func TestHandleNodeFailure(t *testing.T) { }, } - cephfsPV := &corev1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842ca", - Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), - "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, - }, - }, - Spec: corev1.PersistentVolumeSpec{ - PersistentVolumeSource: corev1.PersistentVolumeSource{ - CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), - VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", - VolumeAttributes: map[string]string{ - "fsName": "myfs", - "subvolumePath": "/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", - "subvolumeName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", - }, - }, - }, - }, - } - staticRbdPV := &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb", @@ -278,28 +253,6 @@ func TestHandleNodeFailure(t *testing.T) { }, } - staticCephfsPV := &corev1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842cb", - Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), - "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, - }, - }, - Spec: corev1.PersistentVolumeSpec{ - PersistentVolumeSource: corev1.PersistentVolumeSource{ - CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), - VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", - VolumeAttributes: map[string]string{ - "staticVolume": "true", - }, - }, - }, - }, - } - pvNotProvisionByCSI := &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cc", @@ -343,9 +296,6 @@ func TestHandleNodeFailure(t *testing.T) { _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, rbdPV, metav1.CreateOptions{}) assert.NoError(t, err) - _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, cephfsPV, metav1.CreateOptions{}) - assert.NoError(t, err) - _, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &v1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "networkfences.csiaddons.openshift.io"}}, metav1.CreateOptions{}) assert.NoError(t, err) @@ -357,26 +307,19 @@ func TestHandleNodeFailure(t *testing.T) { err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns)}, networkFenceRbd) assert.NoError(t, err) - networkFenceCephFs := &addonsv1alpha1.NetworkFence{} - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, ns)}, networkFenceCephFs) - assert.NoError(t, err) - networkFences := &addonsv1alpha1.NetworkFenceList{} err = c.client.List(ctx, networkFences) assert.NoError(t, err) - var rbdCount, cephFsCount int + var rbdCount int for _, fence := range networkFences.Items { // Check if the resource is in the desired namespace if strings.Contains(fence.Name, rbdDriver) { rbdCount++ - } else if strings.Contains(fence.Name, cephfsDriver) { - cephFsCount++ } } assert.Equal(t, 1, rbdCount) - assert.Equal(t, 1, cephFsCount) // For static rbd pv _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticRbdPV, metav1.CreateOptions{}) @@ -385,28 +328,13 @@ func TestHandleNodeFailure(t *testing.T) { pvList, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) assert.NoError(t, err) - rbdVolumesInUse, _ := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) + rbdVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse) assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `rbdPV` err = c.handleNodeFailure(ctx, cephCluster, node) assert.NoError(t, err) - // For static cephfs pv - _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticCephfsPV, metav1.CreateOptions{}) - assert.NoError(t, err) - - pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) - assert.NoError(t, err) - - _, cephFSVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) - cephFSVolumesInUseMap := make(map[string]struct{}) - for _, vol := range cephFSVolumesInUse { - cephFSVolumesInUseMap[vol] = struct{}{} - } - cephFSPVList := listRWOCephFSPV(pvList, cephCluster, cephFSVolumesInUseMap) - assert.Equal(t, len(cephFSPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `cephfsPV` - err = c.handleNodeFailure(ctx, cephCluster, node) assert.NoError(t, err) @@ -417,7 +345,7 @@ func TestHandleNodeFailure(t *testing.T) { pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) assert.NoError(t, err) - rbdVolumesInUse, _ = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) + rbdVolumesInUse = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse) assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `PV` @@ -427,15 +355,15 @@ func TestHandleNodeFailure(t *testing.T) { // When out-of-service taint is removed node.Spec.Taints = []corev1.Taint{} + networkFenceRbd.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage + err = c.client.Update(ctx, networkFenceRbd) + assert.NoError(t, err) + err = c.handleNodeFailure(ctx, cephCluster, node) assert.NoError(t, err) err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns), Namespace: cephCluster.Namespace}, networkFenceRbd) assert.Error(t, err, kerrors.IsNotFound(err)) - - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, ns), Namespace: cephCluster.Namespace}, networkFenceCephFs) - assert.Error(t, err, kerrors.IsNotFound(err)) - } func TestGetCephVolumesInUse(t *testing.T) { @@ -443,8 +371,6 @@ func TestGetCephVolumesInUse(t *testing.T) { volInUse := []corev1.UniqueVolumeName{ "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003", - "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", - "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004", } splitVolInUse := trimeVolumeInUse(volInUse[0]) @@ -455,21 +381,11 @@ func TestGetCephVolumesInUse(t *testing.T) { assert.Equal(t, splitVolInUse[0], "rook-ceph.rbd.csi.ceph.com") assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003") - splitVolInUse = trimeVolumeInUse(volInUse[2]) - assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com") - assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001") - - splitVolInUse = trimeVolumeInUse(volInUse[3]) - assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com") - assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004") - - trimRbdVolInUse, trimCephFSVolInUse := getCephVolumesInUse(cephCluster, volInUse) + trimRbdVolInUse := getCephVolumesInUse(cephCluster, volInUse) expectedRbd := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003"} - expectedCephfs := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004"} assert.Equal(t, expectedRbd, trimRbdVolInUse) - assert.Equal(t, expectedCephfs, trimCephFSVolInUse) } func TestRBDStatusUnMarshal(t *testing.T) {