Merge pull request #13806 from subhamkrai/remove-cephfs-fencing-code
csi: remove cephFs networkFence code temporarily
This commit is contained in:
6 files changed
+29
-268
No files matched your search
@@ -10,7 +10,7 @@ require (
|
||||
github.com/banzaicloud/k8s-objectmatcher v1.8.0
|
||||
github.com/ceph/go-ceph v0.24.0
|
||||
github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8
|
||||
github.com/csi-addons/kubernetes-csi-addons v0.7.0
|
||||
github.com/csi-addons/kubernetes-csi-addons v0.8.0
|
||||
github.com/gemalto/kmip-go v0.0.10
|
||||
github.com/go-ini/ini v1.67.0
|
||||
github.com/google/go-cmp v0.6.0
|
||||
|
||||
@@ -167,8 +167,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/csi-addons/kubernetes-csi-addons v0.7.0 h1:Cqon7e9Wnil8gq4/M80uilMJcTv7LypdKJoOVhEb4oM=
|
||||
github.com/csi-addons/kubernetes-csi-addons v0.7.0/go.mod h1:7thZgf/0YB59yWWv8jb0l6mv1bnPEFh5Kxj4k66pnPs=
|
||||
github.com/csi-addons/kubernetes-csi-addons v0.8.0 h1:zvYGp4DM6KdQzEX3dQSYKykqJdLZlxpVBJjtpbaqFjs=
|
||||
github.com/csi-addons/kubernetes-csi-addons v0.8.0/go.mod h1:dvinzoiXlqdOGDpKkYx8Jxl507BzVEEEO+SI0OmBaRI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
@@ -613,7 +613,7 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay
|
||||
github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
|
||||
github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
|
||||
github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc=
|
||||
github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
|
||||
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
@@ -627,7 +627,7 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ
|
||||
github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
|
||||
github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
|
||||
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
|
||||
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
|
||||
github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
|
||||
github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4 h1:5RyeLvTSZEn/fDQA6e6+qIvFPssWjreY8pbwfg4/EEQ=
|
||||
github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU=
|
||||
github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
|
||||
|
||||
+2
-2
@@ -53,8 +53,8 @@ require (
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.11.0 // indirect
|
||||
github.com/onsi/gomega v1.27.10 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
|
||||
github.com/onsi/gomega v1.30.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
|
||||
+4
-4
@@ -407,8 +407,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay
|
||||
github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
|
||||
github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
|
||||
github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc=
|
||||
github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
|
||||
github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
|
||||
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
|
||||
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
@@ -420,8 +420,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ
|
||||
github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
|
||||
github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
|
||||
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
|
||||
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
|
||||
github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
|
||||
github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
|
||||
github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
|
||||
github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4 h1:5RyeLvTSZEn/fDQA6e6+qIvFPssWjreY8pbwfg4/EEQ=
|
||||
github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU=
|
||||
github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
|
||||
|
||||
@@ -60,8 +60,7 @@ var (
|
||||
|
||||
// drivers that supports fencing, used in naming networkFence object
|
||||
const (
|
||||
rbdDriver = "rbd"
|
||||
cephfsDriver = "cephfs"
|
||||
rbdDriver = "rbd"
|
||||
)
|
||||
|
||||
func newClientCluster(client client.Client, namespace string, context *clusterd.Context) *clientCluster {
|
||||
@@ -201,11 +200,6 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C
|
||||
return pkgerror.Wrapf(err, "failed to delete rbd network fence for node %q.", node.Name)
|
||||
}
|
||||
|
||||
err = c.unfenceAndDeleteNetworkFence(ctx, *node, cluster, cephfsDriver)
|
||||
if err != nil {
|
||||
return pkgerror.Wrapf(err, "failed to delete cephFS network fence for node %q.", node.Name)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -217,9 +211,9 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste
|
||||
}
|
||||
logger.Debugf("volumesInuse %s", volumesInuse)
|
||||
|
||||
rbdVolumesInUse, cephFSVolumeInUse := getCephVolumesInUse(cluster, volumesInuse)
|
||||
if len(rbdVolumesInUse) == 0 && len(cephFSVolumeInUse) == 0 {
|
||||
logger.Debugf("no rbd or cephFS volumes in use for out of service node %q", node.Name)
|
||||
rbdVolumesInUse := getCephVolumesInUse(cluster, volumesInuse)
|
||||
if len(rbdVolumesInUse) == 0 {
|
||||
logger.Debugf("no rbd volumes in use for out of service node %q", node.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -258,46 +252,11 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste
|
||||
}
|
||||
}
|
||||
|
||||
if len(cephFSVolumeInUse) != 0 {
|
||||
cephFSVolumeInUseMap := make(map[string]struct{})
|
||||
for _, vol := range cephFSVolumeInUse {
|
||||
cephFSVolumeInUseMap[vol] = struct{}{}
|
||||
}
|
||||
cephFSPVList := listRWOCephFSPV(listPVs, cluster, cephFSVolumeInUseMap)
|
||||
if len(cephFSPVList) == 0 {
|
||||
logger.Debug("No cephFS PVs found on the node")
|
||||
return nil
|
||||
}
|
||||
logger.Infof("node %q require fencing, found cephFS volumes in use", node.Name)
|
||||
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(c.context, ctx, cluster.Namespace, &cluster.Spec)
|
||||
if err != nil {
|
||||
return pkgerror.Wrapf(err, "Failed to load cluster info.")
|
||||
}
|
||||
|
||||
for i := range cephFSPVList {
|
||||
err = c.fenceCephFSVolume(ctx, node, cluster, clusterInfo, cephFSPVList[i])
|
||||
// We only need to create the network fence for any one of cephFS pv.
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
// continue to fence next rbd volume if active client not found
|
||||
if stderrors.Is(err, errActiveClientNotFound) {
|
||||
continue
|
||||
}
|
||||
if i == len(cephFSPVList)-1 {
|
||||
return pkgerror.Wrapf(err, "failed to fence cephFS volumes")
|
||||
}
|
||||
logger.Errorf("failed to fence cephFS volumes %q, trying next cephFS volume", cephFSPVList[i].Name)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) ([]string, []string) {
|
||||
var rbdVolumesInUse, cephFSVolumeInUse []string
|
||||
func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) []string {
|
||||
var rbdVolumesInUse []string
|
||||
|
||||
for _, volume := range volumesInUse {
|
||||
splitVolumeInUseBased := trimeVolumeInUse(volume)
|
||||
@@ -306,13 +265,9 @@ func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.Uniq
|
||||
if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) {
|
||||
rbdVolumesInUse = append(rbdVolumesInUse, splitVolumeInUseBased[1])
|
||||
}
|
||||
|
||||
if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.cephfs.csi.ceph.com", cluster.Namespace) {
|
||||
cephFSVolumeInUse = append(cephFSVolumeInUse, splitVolumeInUseBased[1])
|
||||
}
|
||||
}
|
||||
|
||||
return rbdVolumesInUse, cephFSVolumeInUse
|
||||
return rbdVolumesInUse
|
||||
}
|
||||
|
||||
func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string {
|
||||
@@ -351,36 +306,6 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster
|
||||
return listRbdPV
|
||||
}
|
||||
|
||||
func listRWOCephFSPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, cephFSVolumesInUse map[string]struct{}) []corev1.PersistentVolume {
|
||||
var listCephFSPV []corev1.PersistentVolume
|
||||
|
||||
for _, pv := range listPVs.Items {
|
||||
// Skip if pv is not provisioned by CSI
|
||||
if pv.Spec.CSI == nil {
|
||||
logger.Debugf("pv %q is not provisioned by CSI", pv.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
if pv.Spec.CSI.Driver == fmt.Sprintf("%s.cephfs.csi.ceph.com", cluster.Namespace) {
|
||||
// Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes.
|
||||
if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) {
|
||||
continue
|
||||
}
|
||||
|
||||
if pv.Spec.CSI.VolumeAttributes["staticVolume"] == "true" {
|
||||
logger.Debugf("skipping, static pv %q", pv.Name)
|
||||
continue
|
||||
}
|
||||
// Check if the volume is in use
|
||||
if _, exists := cephFSVolumesInUse[pv.Spec.CSI.VolumeHandle]; exists {
|
||||
listCephFSPV = append(listCephFSPV, pv)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return listCephFSPV
|
||||
}
|
||||
|
||||
// pvSupportsMultiNodeAccess returns true if the PV access modes contain ReadWriteMany or ReadOnlyMany.
|
||||
func pvSupportsMultiNodeAccess(accessModes []corev1.PersistentVolumeAccessMode) bool {
|
||||
for _, accessMode := range accessModes {
|
||||
@@ -423,85 +348,6 @@ func (c *clientCluster) fenceRbdImage(
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *clientCluster) fenceCephFSVolume(
|
||||
ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster,
|
||||
clusterInfo *cephclient.ClusterInfo, cephFSPV corev1.PersistentVolume) error {
|
||||
|
||||
logger.Infof("fencing cephfs volume %q on node %q", cephFSPV.Name, node.Name)
|
||||
|
||||
status, err := cephclient.StatusWithUser(c.context, clusterInfo)
|
||||
if err != nil {
|
||||
return pkgerror.Wrapf(err, "failed to get ceph status for check active mds")
|
||||
}
|
||||
|
||||
var activeMDS string
|
||||
for _, fsRank := range status.Fsmap.ByRank {
|
||||
if fsRank.Status == "up:active" {
|
||||
activeMDS = fsRank.Name
|
||||
}
|
||||
}
|
||||
|
||||
args := []string{"tell", fmt.Sprintf("mds.%s", activeMDS), "client", "ls", "--format", "json"}
|
||||
cmd := cephclient.NewCephCommand(c.context, clusterInfo, args)
|
||||
cmd.JsonOutput = true
|
||||
|
||||
buf, err := cmd.Run()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list watchers for cephfs pool/subvoumeName %s/%s. %v", cephFSPV.Spec.CSI.VolumeAttributes["pool"], cephFSPV.Spec.CSI.VolumeAttributes["subvolumeName"], err)
|
||||
}
|
||||
ips, err := cephFSMDSClientMarshal(buf, cephFSPV)
|
||||
if err != nil {
|
||||
return pkgerror.Wrapf(err, "failed to unmarshal cephfs mds output")
|
||||
}
|
||||
|
||||
if len(ips) == 0 {
|
||||
logger.Infof("no active mds clients found for cephfs volume %q", cephFSPV.Name)
|
||||
return errActiveClientNotFound
|
||||
}
|
||||
|
||||
err = c.createNetworkFence(ctx, cephFSPV, node, cluster, ips, cephfsDriver)
|
||||
if err != nil {
|
||||
return pkgerror.Wrapf(err, "failed to create network fence for node %q", node.Name)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cephFSMDSClientMarshal(output []byte, cephFSPV corev1.PersistentVolume) ([]string, error) {
|
||||
type entity struct {
|
||||
Addr struct {
|
||||
Addr string `json:"addr"`
|
||||
Nonce int `json:"nonce"`
|
||||
} `json:"addr"`
|
||||
}
|
||||
|
||||
type clientMetadata struct {
|
||||
Root string `json:"root"`
|
||||
}
|
||||
|
||||
type cephFSData struct {
|
||||
Entity entity `json:"entity"`
|
||||
ClientMetadata clientMetadata `json:"client_metadata"`
|
||||
}
|
||||
|
||||
var data []cephFSData
|
||||
err := json.Unmarshal([]byte(output), &data)
|
||||
if err != nil {
|
||||
return []string{}, pkgerror.Wrapf(err, "failed to unmarshal cephFS data output")
|
||||
}
|
||||
|
||||
watcherIPlist := []string{}
|
||||
for _, d := range data {
|
||||
if cephFSPV.Spec.CSI.VolumeAttributes["subvolumePath"] == d.ClientMetadata.Root {
|
||||
logger.Infof("cephfs mds client ips to fence %v", d.Entity.Addr)
|
||||
watcherIP := concatenateWatcherIp(d.Entity.Addr.Addr)
|
||||
watcherIPlist = append(watcherIPlist, watcherIP)
|
||||
}
|
||||
}
|
||||
|
||||
return watcherIPlist, nil
|
||||
}
|
||||
|
||||
func rbdStatusUnMarshal(output []byte) ([]string, error) {
|
||||
type rbdStatus struct {
|
||||
Watchers []struct {
|
||||
@@ -524,8 +370,7 @@ func rbdStatusUnMarshal(output []byte) ([]string, error) {
|
||||
}
|
||||
|
||||
func concatenateWatcherIp(address string) string {
|
||||
// address is in format `10.63.0.5:0/1254753579` for rbd and
|
||||
// in the format '10.244.0.12:0' for cephfs
|
||||
// address is in format `10.63.0.5:0/1254753579` for rbd
|
||||
// split with separation ':0' to remove nounce and concatenating `/32` to define a network with only one IP address
|
||||
watcherIP := strings.Split(address, ":0")[0] + "/32"
|
||||
return watcherIP
|
||||
@@ -604,8 +449,8 @@ func (c *clientCluster) unfenceAndDeleteNetworkFence(ctx context.Context, node c
|
||||
return false, err
|
||||
}
|
||||
|
||||
if networkFence.Spec.FenceState != addonsv1alpha1.Unfenced {
|
||||
logger.Infof("waiting for network fence CR %s to get in %s state before deletion", networkFence.Name, addonsv1alpha1.Unfenced)
|
||||
if networkFence.Status.Message != addonsv1alpha1.UnFenceOperationSuccessfulMessage {
|
||||
logger.Infof("waiting for network fence CR %q status to get result %q", networkFence.Name, addonsv1alpha1.UnFenceOperationSuccessfulMessage)
|
||||
return false, err
|
||||
}
|
||||
|
||||
|
||||
@@ -206,7 +206,6 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
},
|
||||
VolumesInUse: []corev1.UniqueVolumeName{
|
||||
"kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
|
||||
"kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -234,30 +233,6 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
cephfsPV := &corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842ca",
|
||||
Annotations: map[string]string{
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", ns),
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner",
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": ns,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PersistentVolumeSpec{
|
||||
PersistentVolumeSource: corev1.PersistentVolumeSource{
|
||||
CSI: &corev1.CSIPersistentVolumeSource{
|
||||
Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", ns),
|
||||
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
|
||||
VolumeAttributes: map[string]string{
|
||||
"fsName": "myfs",
|
||||
"subvolumePath": "/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca",
|
||||
"subvolumeName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
staticRbdPV := &corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb",
|
||||
@@ -278,28 +253,6 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
staticCephfsPV := &corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842cb",
|
||||
Annotations: map[string]string{
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", ns),
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner",
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": ns,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PersistentVolumeSpec{
|
||||
PersistentVolumeSource: corev1.PersistentVolumeSource{
|
||||
CSI: &corev1.CSIPersistentVolumeSource{
|
||||
Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", ns),
|
||||
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
|
||||
VolumeAttributes: map[string]string{
|
||||
"staticVolume": "true",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pvNotProvisionByCSI := &corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cc",
|
||||
@@ -343,9 +296,6 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, rbdPV, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, cephfsPV, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &v1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "networkfences.csiaddons.openshift.io"}}, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -357,26 +307,19 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns)}, networkFenceRbd)
|
||||
assert.NoError(t, err)
|
||||
|
||||
networkFenceCephFs := &addonsv1alpha1.NetworkFence{}
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, ns)}, networkFenceCephFs)
|
||||
assert.NoError(t, err)
|
||||
|
||||
networkFences := &addonsv1alpha1.NetworkFenceList{}
|
||||
err = c.client.List(ctx, networkFences)
|
||||
assert.NoError(t, err)
|
||||
var rbdCount, cephFsCount int
|
||||
var rbdCount int
|
||||
|
||||
for _, fence := range networkFences.Items {
|
||||
// Check if the resource is in the desired namespace
|
||||
if strings.Contains(fence.Name, rbdDriver) {
|
||||
rbdCount++
|
||||
} else if strings.Contains(fence.Name, cephfsDriver) {
|
||||
cephFsCount++
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, rbdCount)
|
||||
assert.Equal(t, 1, cephFsCount)
|
||||
|
||||
// For static rbd pv
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticRbdPV, metav1.CreateOptions{})
|
||||
@@ -385,28 +328,13 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
pvList, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
rbdVolumesInUse, _ := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse)
|
||||
rbdVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse)
|
||||
rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse)
|
||||
assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `rbdPV`
|
||||
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// For static cephfs pv
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticCephfsPV, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, cephFSVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse)
|
||||
cephFSVolumesInUseMap := make(map[string]struct{})
|
||||
for _, vol := range cephFSVolumesInUse {
|
||||
cephFSVolumesInUseMap[vol] = struct{}{}
|
||||
}
|
||||
cephFSPVList := listRWOCephFSPV(pvList, cephCluster, cephFSVolumesInUseMap)
|
||||
assert.Equal(t, len(cephFSPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `cephfsPV`
|
||||
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node)
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -417,7 +345,7 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
rbdVolumesInUse, _ = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse)
|
||||
rbdVolumesInUse = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse)
|
||||
rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse)
|
||||
assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `PV`
|
||||
|
||||
@@ -427,15 +355,15 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
// When out-of-service taint is removed
|
||||
node.Spec.Taints = []corev1.Taint{}
|
||||
|
||||
networkFenceRbd.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage
|
||||
err = c.client.Update(ctx, networkFenceRbd)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns), Namespace: cephCluster.Namespace}, networkFenceRbd)
|
||||
assert.Error(t, err, kerrors.IsNotFound(err))
|
||||
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, ns), Namespace: cephCluster.Namespace}, networkFenceCephFs)
|
||||
assert.Error(t, err, kerrors.IsNotFound(err))
|
||||
|
||||
}
|
||||
|
||||
func TestGetCephVolumesInUse(t *testing.T) {
|
||||
@@ -443,8 +371,6 @@ func TestGetCephVolumesInUse(t *testing.T) {
|
||||
volInUse := []corev1.UniqueVolumeName{
|
||||
"kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
|
||||
"kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003",
|
||||
"kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
|
||||
"kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004",
|
||||
}
|
||||
|
||||
splitVolInUse := trimeVolumeInUse(volInUse[0])
|
||||
@@ -455,21 +381,11 @@ func TestGetCephVolumesInUse(t *testing.T) {
|
||||
assert.Equal(t, splitVolInUse[0], "rook-ceph.rbd.csi.ceph.com")
|
||||
assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003")
|
||||
|
||||
splitVolInUse = trimeVolumeInUse(volInUse[2])
|
||||
assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com")
|
||||
assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001")
|
||||
|
||||
splitVolInUse = trimeVolumeInUse(volInUse[3])
|
||||
assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com")
|
||||
assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004")
|
||||
|
||||
trimRbdVolInUse, trimCephFSVolInUse := getCephVolumesInUse(cephCluster, volInUse)
|
||||
trimRbdVolInUse := getCephVolumesInUse(cephCluster, volInUse)
|
||||
|
||||
expectedRbd := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003"}
|
||||
expectedCephfs := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004"}
|
||||
|
||||
assert.Equal(t, expectedRbd, trimRbdVolInUse)
|
||||
assert.Equal(t, expectedCephfs, trimCephFSVolInUse)
|
||||
}
|
||||
|
||||
func TestRBDStatusUnMarshal(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user