From efe6ef4e451005b082318019dc22e75076e510d3 Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Fri, 17 May 2024 12:04:17 +0530 Subject: [PATCH 1/2] csi: revert remove cephFs networkFence code temporarily This reverts commit bd4250f56e202daf9cf40ae3dd16382cb75663df. Signed-off-by: subhamkrai --- pkg/operator/ceph/cluster/watcher.go | 171 +++++++++++++++++++++- pkg/operator/ceph/cluster/watcher_test.go | 100 ++++++++++++- 2 files changed, 255 insertions(+), 16 deletions(-) diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index a79c71221..79aac4fe7 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -60,7 +60,8 @@ var ( // drivers that supports fencing, used in naming networkFence object const ( - rbdDriver = "rbd" + rbdDriver = "rbd" + cephfsDriver = "cephfs" ) func newClientCluster(client client.Client, namespace string, context *clusterd.Context) *clientCluster { @@ -210,6 +211,11 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C return pkgerror.Wrapf(err, "failed to delete rbd network fence for node %q.", node.Name) } + err = c.unfenceAndDeleteNetworkFence(ctx, *node, cluster, cephfsDriver) + if err != nil { + return pkgerror.Wrapf(err, "failed to delete cephFS network fence for node %q.", node.Name) + } + return nil } @@ -221,9 +227,9 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste } logger.Debugf("volumesInuse %s", volumesInuse) - rbdVolumesInUse := getCephVolumesInUse(cluster, volumesInuse) - if len(rbdVolumesInUse) == 0 { - logger.Debugf("no rbd volumes in use for out of service node %q", node.Name) + rbdVolumesInUse, cephFSVolumeInUse := getCephVolumesInUse(cluster, volumesInuse) + if len(rbdVolumesInUse) == 0 && len(cephFSVolumeInUse) == 0 { + logger.Debugf("no rbd or cephFS volumes in use for out of service node %q", node.Name) return nil } @@ -262,11 +268,46 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste } } + if len(cephFSVolumeInUse) != 0 { + cephFSVolumeInUseMap := make(map[string]struct{}) + for _, vol := range cephFSVolumeInUse { + cephFSVolumeInUseMap[vol] = struct{}{} + } + cephFSPVList := listRWOCephFSPV(listPVs, cluster, cephFSVolumeInUseMap) + if len(cephFSPVList) == 0 { + logger.Debug("No cephFS PVs found on the node") + return nil + } + logger.Infof("node %q require fencing, found cephFS volumes in use", node.Name) + clusterInfo, _, _, err := opcontroller.LoadClusterInfo(c.context, ctx, cluster.Namespace, &cluster.Spec) + if err != nil { + return pkgerror.Wrapf(err, "Failed to load cluster info.") + } + + for i := range cephFSPVList { + err = c.fenceCephFSVolume(ctx, node, cluster, clusterInfo, cephFSPVList[i]) + // We only need to create the network fence for any one of cephFS pv. + if err == nil { + break + } + + // continue to fence next rbd volume if active client not found + if stderrors.Is(err, errActiveClientNotFound) { + continue + } + if i == len(cephFSPVList)-1 { + return pkgerror.Wrapf(err, "failed to fence cephFS volumes") + } + logger.Errorf("failed to fence cephFS volumes %q, trying next cephFS volume", cephFSPVList[i].Name) + } + + } + return nil } -func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) []string { - var rbdVolumesInUse []string +func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) ([]string, []string) { + var rbdVolumesInUse, cephFSVolumeInUse []string for _, volume := range volumesInUse { splitVolumeInUseBased := trimeVolumeInUse(volume) @@ -275,9 +316,13 @@ func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.Uniq if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) { rbdVolumesInUse = append(rbdVolumesInUse, splitVolumeInUseBased[1]) } + + if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.cephfs.csi.ceph.com", cluster.Namespace) { + cephFSVolumeInUse = append(cephFSVolumeInUse, splitVolumeInUseBased[1]) + } } - return rbdVolumesInUse + return rbdVolumesInUse, cephFSVolumeInUse } func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string { @@ -316,6 +361,36 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster return listRbdPV } +func listRWOCephFSPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, cephFSVolumesInUse map[string]struct{}) []corev1.PersistentVolume { + var listCephFSPV []corev1.PersistentVolume + + for _, pv := range listPVs.Items { + // Skip if pv is not provisioned by CSI + if pv.Spec.CSI == nil { + logger.Debugf("pv %q is not provisioned by CSI", pv.Name) + continue + } + + if pv.Spec.CSI.Driver == fmt.Sprintf("%s.cephfs.csi.ceph.com", cluster.Namespace) { + // Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes. + if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) { + continue + } + + if pv.Spec.CSI.VolumeAttributes["staticVolume"] == "true" { + logger.Debugf("skipping, static pv %q", pv.Name) + continue + } + // Check if the volume is in use + if _, exists := cephFSVolumesInUse[pv.Spec.CSI.VolumeHandle]; exists { + listCephFSPV = append(listCephFSPV, pv) + } + } + + } + return listCephFSPV +} + // pvSupportsMultiNodeAccess returns true if the PV access modes contain ReadWriteMany or ReadOnlyMany. func pvSupportsMultiNodeAccess(accessModes []corev1.PersistentVolumeAccessMode) bool { for _, accessMode := range accessModes { @@ -358,6 +433,85 @@ func (c *clientCluster) fenceRbdImage( return nil } +func (c *clientCluster) fenceCephFSVolume( + ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster, + clusterInfo *cephclient.ClusterInfo, cephFSPV corev1.PersistentVolume) error { + + logger.Infof("fencing cephfs volume %q on node %q", cephFSPV.Name, node.Name) + + status, err := cephclient.StatusWithUser(c.context, clusterInfo) + if err != nil { + return pkgerror.Wrapf(err, "failed to get ceph status for check active mds") + } + + var activeMDS string + for _, fsRank := range status.Fsmap.ByRank { + if fsRank.Status == "up:active" { + activeMDS = fsRank.Name + } + } + + args := []string{"tell", fmt.Sprintf("mds.%s", activeMDS), "client", "ls", "--format", "json"} + cmd := cephclient.NewCephCommand(c.context, clusterInfo, args) + cmd.JsonOutput = true + + buf, err := cmd.Run() + if err != nil { + return fmt.Errorf("failed to list watchers for cephfs pool/subvoumeName %s/%s. %v", cephFSPV.Spec.CSI.VolumeAttributes["pool"], cephFSPV.Spec.CSI.VolumeAttributes["subvolumeName"], err) + } + ips, err := cephFSMDSClientMarshal(buf, cephFSPV) + if err != nil { + return pkgerror.Wrapf(err, "failed to unmarshal cephfs mds output") + } + + if len(ips) == 0 { + logger.Infof("no active mds clients found for cephfs volume %q", cephFSPV.Name) + return errActiveClientNotFound + } + + err = c.createNetworkFence(ctx, cephFSPV, node, cluster, ips, cephfsDriver) + if err != nil { + return pkgerror.Wrapf(err, "failed to create network fence for node %q", node.Name) + } + + return nil +} + +func cephFSMDSClientMarshal(output []byte, cephFSPV corev1.PersistentVolume) ([]string, error) { + type entity struct { + Addr struct { + Addr string `json:"addr"` + Nonce int `json:"nonce"` + } `json:"addr"` + } + + type clientMetadata struct { + Root string `json:"root"` + } + + type cephFSData struct { + Entity entity `json:"entity"` + ClientMetadata clientMetadata `json:"client_metadata"` + } + + var data []cephFSData + err := json.Unmarshal([]byte(output), &data) + if err != nil { + return []string{}, pkgerror.Wrapf(err, "failed to unmarshal cephFS data output") + } + + watcherIPlist := []string{} + for _, d := range data { + if cephFSPV.Spec.CSI.VolumeAttributes["subvolumePath"] == d.ClientMetadata.Root { + logger.Infof("cephfs mds client ips to fence %v", d.Entity.Addr) + watcherIP := concatenateWatcherIp(d.Entity.Addr.Addr) + watcherIPlist = append(watcherIPlist, watcherIP) + } + } + + return watcherIPlist, nil +} + func rbdStatusUnMarshal(output []byte) ([]string, error) { type rbdStatus struct { Watchers []struct { @@ -380,7 +534,8 @@ func rbdStatusUnMarshal(output []byte) ([]string, error) { } func concatenateWatcherIp(address string) string { - // address is in format `10.63.0.5:0/1254753579` for rbd + // address is in format `10.63.0.5:0/1254753579` for rbd and + // in the format '10.244.0.12:0' for cephfs // split with separation ':0' to remove nounce and concatenating `/32` to define a network with only one IP address watcherIP := strings.Split(address, ":0")[0] + "/32" return watcherIP diff --git a/pkg/operator/ceph/cluster/watcher_test.go b/pkg/operator/ceph/cluster/watcher_test.go index e3585ae81..29b6d60f8 100644 --- a/pkg/operator/ceph/cluster/watcher_test.go +++ b/pkg/operator/ceph/cluster/watcher_test.go @@ -206,6 +206,7 @@ func TestHandleNodeFailure(t *testing.T) { }, VolumesInUse: []corev1.UniqueVolumeName{ "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", + "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", }, }, } @@ -233,6 +234,30 @@ func TestHandleNodeFailure(t *testing.T) { }, } + cephfsPV := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842ca", + Annotations: map[string]string{ + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), + "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", + "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), + VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", + VolumeAttributes: map[string]string{ + "fsName": "myfs", + "subvolumePath": "/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", + "subvolumeName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", + }, + }, + }, + }, + } + staticRbdPV := &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb", @@ -253,6 +278,28 @@ func TestHandleNodeFailure(t *testing.T) { }, } + staticCephfsPV := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842cb", + Annotations: map[string]string{ + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), + "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", + "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), + VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", + VolumeAttributes: map[string]string{ + "staticVolume": "true", + }, + }, + }, + }, + } + pvNotProvisionByCSI := &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cc", @@ -296,6 +343,9 @@ func TestHandleNodeFailure(t *testing.T) { _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, rbdPV, metav1.CreateOptions{}) assert.NoError(t, err) + _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, cephfsPV, metav1.CreateOptions{}) + assert.NoError(t, err) + _, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &v1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "networkfences.csiaddons.openshift.io"}}, metav1.CreateOptions{}) assert.NoError(t, err) @@ -307,19 +357,26 @@ func TestHandleNodeFailure(t *testing.T) { err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns)}, networkFenceRbd) assert.NoError(t, err) + networkFenceCephFs := &addonsv1alpha1.NetworkFence{} + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, ns)}, networkFenceCephFs) + assert.NoError(t, err) + networkFences := &addonsv1alpha1.NetworkFenceList{} err = c.client.List(ctx, networkFences) assert.NoError(t, err) - var rbdCount int + var rbdCount, cephFsCount int for _, fence := range networkFences.Items { // Check if the resource is in the desired namespace if strings.Contains(fence.Name, rbdDriver) { rbdCount++ + } else if strings.Contains(fence.Name, cephfsDriver) { + cephFsCount++ } } assert.Equal(t, 1, rbdCount) + assert.Equal(t, 1, cephFsCount) // For static rbd pv _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticRbdPV, metav1.CreateOptions{}) @@ -328,13 +385,28 @@ func TestHandleNodeFailure(t *testing.T) { pvList, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) assert.NoError(t, err) - rbdVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) + rbdVolumesInUse, _ := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse) assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `rbdPV` err = c.handleNodeFailure(ctx, cephCluster, node) assert.NoError(t, err) + // For static cephfs pv + _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticCephfsPV, metav1.CreateOptions{}) + assert.NoError(t, err) + + pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + + _, cephFSVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) + cephFSVolumesInUseMap := make(map[string]struct{}) + for _, vol := range cephFSVolumesInUse { + cephFSVolumesInUseMap[vol] = struct{}{} + } + cephFSPVList := listRWOCephFSPV(pvList, cephCluster, cephFSVolumesInUseMap) + assert.Equal(t, len(cephFSPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `cephfsPV` + err = c.handleNodeFailure(ctx, cephCluster, node) assert.NoError(t, err) @@ -345,7 +417,7 @@ func TestHandleNodeFailure(t *testing.T) { pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) assert.NoError(t, err) - rbdVolumesInUse = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) + rbdVolumesInUse, _ = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse) assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `PV` @@ -355,15 +427,15 @@ func TestHandleNodeFailure(t *testing.T) { // When out-of-service taint is removed node.Spec.Taints = []corev1.Taint{} - networkFenceRbd.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage - err = c.client.Update(ctx, networkFenceRbd) - assert.NoError(t, err) - err = c.handleNodeFailure(ctx, cephCluster, node) assert.NoError(t, err) err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns), Namespace: cephCluster.Namespace}, networkFenceRbd) assert.Error(t, err, kerrors.IsNotFound(err)) + + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, ns), Namespace: cephCluster.Namespace}, networkFenceCephFs) + assert.Error(t, err, kerrors.IsNotFound(err)) + } func TestGetCephVolumesInUse(t *testing.T) { @@ -371,6 +443,8 @@ func TestGetCephVolumesInUse(t *testing.T) { volInUse := []corev1.UniqueVolumeName{ "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003", + "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", + "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004", } splitVolInUse := trimeVolumeInUse(volInUse[0]) @@ -381,11 +455,21 @@ func TestGetCephVolumesInUse(t *testing.T) { assert.Equal(t, splitVolInUse[0], "rook-ceph.rbd.csi.ceph.com") assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003") - trimRbdVolInUse := getCephVolumesInUse(cephCluster, volInUse) + splitVolInUse = trimeVolumeInUse(volInUse[2]) + assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com") + assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001") + + splitVolInUse = trimeVolumeInUse(volInUse[3]) + assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com") + assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004") + + trimRbdVolInUse, trimCephFSVolInUse := getCephVolumesInUse(cephCluster, volInUse) expectedRbd := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003"} + expectedCephfs := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004"} assert.Equal(t, expectedRbd, trimRbdVolInUse) + assert.Equal(t, expectedCephfs, trimCephFSVolInUse) } func TestRBDStatusUnMarshal(t *testing.T) { From 9f1b64be54eb62a4f1d36f35cc6ca839c31de61a Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Thu, 23 May 2024 18:18:46 +0530 Subject: [PATCH 2/2] mds: block of active mds ip only currently, it was blocking both the mds ip instead its should block ip of of mds which is in the same node which is down. Signed-off-by: subhamkrai --- pkg/operator/ceph/cluster/controller.go | 12 +-- pkg/operator/ceph/cluster/predicate.go | 6 +- pkg/operator/ceph/cluster/watcher.go | 84 ++++++++++---------- pkg/operator/ceph/cluster/watcher_test.go | 93 +++++++++++++---------- pkg/operator/ceph/cr_manager.go | 6 +- 5 files changed, 106 insertions(+), 95 deletions(-) diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 6466607ea..4638f2bab 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -99,16 +99,17 @@ type ReconcileCephCluster struct { context *clusterd.Context clusterController *ClusterController opManagerContext context.Context + opConfig opcontroller.OperatorConfig } // Add creates a new CephCluster Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) error { - return add(opManagerContext, mgr, newReconciler(mgr, ctx, clusterController, opManagerContext), ctx) +func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(opManagerContext, mgr, newReconciler(mgr, ctx, clusterController, opManagerContext, opConfig), ctx, opConfig) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { // add "rook-" prefix to the controller name to make sure it is clear to all reading the events // that they are coming from Rook. The controller name already has context that it is for Ceph // and from the cluster controller. @@ -118,12 +119,13 @@ func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController client: mgr.GetClient(), scheme: mgr.GetScheme(), context: ctx, + opConfig: opConfig, clusterController: clusterController, opManagerContext: opManagerContext, } } -func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context) error { +func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context, opConfig opcontroller.OperatorConfig) error { // Create a new controller c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r}) if err != nil { @@ -170,7 +172,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco mgr.GetCache(), &corev1.Node{TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: corev1.SchemeGroupVersion.String()}}, handler.EnqueueRequestsFromMapFunc(handlerFunc), - predicateForNodeWatcher(opManagerContext, mgr.GetClient(), context)) + predicateForNodeWatcher(opManagerContext, mgr.GetClient(), context, opConfig.OperatorNamespace)) err = c.Watch(nodeKind) if err != nil { return err diff --git a/pkg/operator/ceph/cluster/predicate.go b/pkg/operator/ceph/cluster/predicate.go index f7f5112a6..709b5b6bf 100644 --- a/pkg/operator/ceph/cluster/predicate.go +++ b/pkg/operator/ceph/cluster/predicate.go @@ -52,11 +52,11 @@ func shouldReconcileChangedNode(objOld, objNew *corev1.Node) bool { } // predicateForNodeWatcher is the predicate function to trigger reconcile on Node events -func predicateForNodeWatcher(ctx context.Context, client client.Client, context *clusterd.Context) predicate.Funcs { +func predicateForNodeWatcher(ctx context.Context, client client.Client, context *clusterd.Context, opNamespace string) predicate.Funcs { return predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { clientCluster := newClientCluster(client, e.Object.GetNamespace(), context) - return clientCluster.onK8sNode(ctx, e.Object) + return clientCluster.onK8sNode(ctx, e.Object, opNamespace) }, UpdateFunc: func(e event.UpdateEvent) bool { @@ -67,7 +67,7 @@ func predicateForNodeWatcher(ctx context.Context, client client.Client, context } clientCluster := newClientCluster(client, e.ObjectNew.GetNamespace(), context) - return clientCluster.onK8sNode(ctx, e.ObjectNew) + return clientCluster.onK8sNode(ctx, e.ObjectNew, opNamespace) } } return false diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index 79aac4fe7..f57021dde 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -81,7 +81,7 @@ func checkStorageForNode(cluster *cephv1.CephCluster) bool { } // onK8sNode is triggered when a node is added in the Kubernetes cluster -func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bool { +func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, opNamespace string) bool { node, ok := object.(*corev1.Node) if !ok { return false @@ -91,7 +91,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bo cluster := c.getCephCluster() // Continue reconcile in case of failure too since we don't want to block other node reconcile - if err := c.handleNodeFailure(ctx, cluster, node); err != nil { + if err := c.handleNodeFailure(ctx, cluster, node, opNamespace); err != nil { logger.Errorf("failed to handle node failure. %v", err) } @@ -158,7 +158,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bo return false } -func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node) error { +func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node, opNamespace string) error { watchForNodeLoss, err := k8sutil.GetOperatorSetting(ctx, c.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_WATCH_FOR_NODE_FAILURE", "true") if err != nil { return pkgerror.Wrapf(err, "failed to get configmap value `ROOK_WATCH_FOR_NODE_FAILURE`.") @@ -199,7 +199,7 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C } if nodeHasOutOfServiceTaint { - err := c.fenceNode(ctx, node, cluster) + err := c.fenceNode(ctx, node, cluster, opNamespace) if err != nil { return pkgerror.Wrapf(err, "failed to create network fence for node %q.", node.Name) } @@ -215,11 +215,10 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C if err != nil { return pkgerror.Wrapf(err, "failed to delete cephFS network fence for node %q.", node.Name) } - return nil } -func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster) error { +func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster, opNamespace string) error { volumesInuse := node.Status.VolumesInUse if len(volumesInuse) == 0 { logger.Debugf("no volumes in use for node %q", node.Name) @@ -227,9 +226,9 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste } logger.Debugf("volumesInuse %s", volumesInuse) - rbdVolumesInUse, cephFSVolumeInUse := getCephVolumesInUse(cluster, volumesInuse) + rbdVolumesInUse, cephFSVolumeInUse := getCephVolumesInUse(cluster, volumesInuse, opNamespace) if len(rbdVolumesInUse) == 0 && len(cephFSVolumeInUse) == 0 { - logger.Debugf("no rbd or cephFS volumes in use for out of service node %q", node.Name) + logger.Debugf("no rbd or cephfs subvolumes in use for out of service node %q", node.Name) return nil } @@ -239,7 +238,7 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste } if len(rbdVolumesInUse) != 0 { - rbdPVList := listRBDPV(listPVs, cluster, rbdVolumesInUse) + rbdPVList := listRBDPV(listPVs, cluster, rbdVolumesInUse, opNamespace) if len(rbdPVList) == 0 { logger.Debug("No rbd PVs found on the node") } else { @@ -273,55 +272,53 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste for _, vol := range cephFSVolumeInUse { cephFSVolumeInUseMap[vol] = struct{}{} } - cephFSPVList := listRWOCephFSPV(listPVs, cluster, cephFSVolumeInUseMap) + cephFSPVList := listRWOCephFSPV(listPVs, cluster, cephFSVolumeInUseMap, opNamespace) if len(cephFSPVList) == 0 { - logger.Debug("No cephFS PVs found on the node") + logger.Debug("No cephfs PVs found on the node %s", node.Name) return nil } - logger.Infof("node %q require fencing, found cephFS volumes in use", node.Name) + logger.Infof("node %q require fencing, found cephfs subvolumes in use", node.Name) clusterInfo, _, _, err := opcontroller.LoadClusterInfo(c.context, ctx, cluster.Namespace, &cluster.Spec) if err != nil { - return pkgerror.Wrapf(err, "Failed to load cluster info.") + return pkgerror.Wrap(err, "Failed to load cluster info.") } for i := range cephFSPVList { - err = c.fenceCephFSVolume(ctx, node, cluster, clusterInfo, cephFSPVList[i]) + err = c.fenceCephFSSubvolume(ctx, node, cluster, clusterInfo, cephFSPVList[i]) // We only need to create the network fence for any one of cephFS pv. if err == nil { break } - // continue to fence next rbd volume if active client not found + // continue to fence next cephfs subvolume if active client not found if stderrors.Is(err, errActiveClientNotFound) { continue } if i == len(cephFSPVList)-1 { - return pkgerror.Wrapf(err, "failed to fence cephFS volumes") + return pkgerror.Wrap(err, "failed to fence cephfs subvolumes") } - logger.Errorf("failed to fence cephFS volumes %q, trying next cephFS volume", cephFSPVList[i].Name) + logger.Errorf("failed to fence cephfs subvolume %q, trying next cephfs subvolume", cephFSPVList[i].Name) } } - return nil } -func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) ([]string, []string) { +func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName, opNamespace string) ([]string, []string) { var rbdVolumesInUse, cephFSVolumeInUse []string for _, volume := range volumesInUse { splitVolumeInUseBased := trimeVolumeInUse(volume) logger.Infof("volumeInUse after split based on '^' %v", splitVolumeInUseBased) - if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) { + if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", opNamespace) { rbdVolumesInUse = append(rbdVolumesInUse, splitVolumeInUseBased[1]) } - if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.cephfs.csi.ceph.com", cluster.Namespace) { + if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.cephfs.csi.ceph.com", opNamespace) { cephFSVolumeInUse = append(cephFSVolumeInUse, splitVolumeInUseBased[1]) } } - return rbdVolumesInUse, cephFSVolumeInUse } @@ -331,7 +328,7 @@ func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string { return splitVolumeInUseBased } -func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, rbdVolumesInUse []string) []corev1.PersistentVolume { +func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, rbdVolumesInUse []string, opNamespace string) []corev1.PersistentVolume { var listRbdPV []corev1.PersistentVolume for _, pv := range listPVs.Items { @@ -341,7 +338,7 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster continue } - if pv.Spec.CSI.Driver == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) { + if pv.Spec.CSI.Driver == fmt.Sprintf("%s.rbd.csi.ceph.com", opNamespace) { // Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes. if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) { continue @@ -361,24 +358,24 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster return listRbdPV } -func listRWOCephFSPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, cephFSVolumesInUse map[string]struct{}) []corev1.PersistentVolume { +func listRWOCephFSPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, cephFSVolumesInUse map[string]struct{}, opNamespace string) []corev1.PersistentVolume { var listCephFSPV []corev1.PersistentVolume for _, pv := range listPVs.Items { // Skip if pv is not provisioned by CSI if pv.Spec.CSI == nil { - logger.Debugf("pv %q is not provisioned by CSI", pv.Name) + logger.Debugf("PV %q is not provisioned by CSI", pv.Name) continue } - if pv.Spec.CSI.Driver == fmt.Sprintf("%s.cephfs.csi.ceph.com", cluster.Namespace) { + if pv.Spec.CSI.Driver == fmt.Sprintf("%s.cephfs.csi.ceph.com", opNamespace) { // Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes. if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) { continue } if pv.Spec.CSI.VolumeAttributes["staticVolume"] == "true" { - logger.Debugf("skipping, static pv %q", pv.Name) + logger.Debugf("skipping, static PV %q", pv.Name) continue } // Check if the volume is in use @@ -386,7 +383,6 @@ func listRWOCephFSPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephC listCephFSPV = append(listCephFSPV, pv) } } - } return listCephFSPV } @@ -433,15 +429,15 @@ func (c *clientCluster) fenceRbdImage( return nil } -func (c *clientCluster) fenceCephFSVolume( +func (c *clientCluster) fenceCephFSSubvolume( ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster, clusterInfo *cephclient.ClusterInfo, cephFSPV corev1.PersistentVolume) error { - logger.Infof("fencing cephfs volume %q on node %q", cephFSPV.Name, node.Name) + logger.Infof("fencing cephfs subvolume %q on node %q", cephFSPV.Name, node.Name) status, err := cephclient.StatusWithUser(c.context, clusterInfo) if err != nil { - return pkgerror.Wrapf(err, "failed to get ceph status for check active mds") + return pkgerror.Wrap(err, "failed to get ceph status for check active mds") } var activeMDS string @@ -457,15 +453,15 @@ func (c *clientCluster) fenceCephFSVolume( buf, err := cmd.Run() if err != nil { - return fmt.Errorf("failed to list watchers for cephfs pool/subvoumeName %s/%s. %v", cephFSPV.Spec.CSI.VolumeAttributes["pool"], cephFSPV.Spec.CSI.VolumeAttributes["subvolumeName"], err) + return fmt.Errorf("failed to list watchers for cephfs subvolumeName %s. %v", cephFSPV.Spec.CSI.VolumeAttributes["subvolumeName"], err) } - ips, err := cephFSMDSClientMarshal(buf, cephFSPV) + ips, err := cephFSMDSClientMarshal(buf, node.Name, cephFSPV) if err != nil { - return pkgerror.Wrapf(err, "failed to unmarshal cephfs mds output") + return pkgerror.Wrap(err, "failed to unmarshal cephfs mds output") } if len(ips) == 0 { - logger.Infof("no active mds clients found for cephfs volume %q", cephFSPV.Name) + logger.Infof("no active mds clients found for cephfs subvolume %q", cephFSPV.Name) return errActiveClientNotFound } @@ -477,7 +473,7 @@ func (c *clientCluster) fenceCephFSVolume( return nil } -func cephFSMDSClientMarshal(output []byte, cephFSPV corev1.PersistentVolume) ([]string, error) { +func cephFSMDSClientMarshal(output []byte, nodeName string, cephFSPV corev1.PersistentVolume) ([]string, error) { type entity struct { Addr struct { Addr string `json:"addr"` @@ -486,7 +482,8 @@ func cephFSMDSClientMarshal(output []byte, cephFSPV corev1.PersistentVolume) ([] } type clientMetadata struct { - Root string `json:"root"` + Root string `json:"root"` + Hostname string `json:"hostname"` } type cephFSData struct { @@ -502,10 +499,13 @@ func cephFSMDSClientMarshal(output []byte, cephFSPV corev1.PersistentVolume) ([] watcherIPlist := []string{} for _, d := range data { - if cephFSPV.Spec.CSI.VolumeAttributes["subvolumePath"] == d.ClientMetadata.Root { - logger.Infof("cephfs mds client ips to fence %v", d.Entity.Addr) - watcherIP := concatenateWatcherIp(d.Entity.Addr.Addr) - watcherIPlist = append(watcherIPlist, watcherIP) + if d.ClientMetadata.Hostname != "" { + if strings.Contains(nodeName, d.ClientMetadata.Hostname) && cephFSPV.Spec.CSI.VolumeAttributes["subvolumePath"] == d.ClientMetadata.Root { + logger.Infof("cephfs mds client ips to fence %v", d.Entity.Addr) + watcherIP := concatenateWatcherIp(d.Entity.Addr.Addr) + watcherIPlist = append(watcherIPlist, watcherIP) + break + } } } diff --git a/pkg/operator/ceph/cluster/watcher_test.go b/pkg/operator/ceph/cluster/watcher_test.go index 29b6d60f8..f4b83ca07 100644 --- a/pkg/operator/ceph/cluster/watcher_test.go +++ b/pkg/operator/ceph/cluster/watcher_test.go @@ -96,6 +96,7 @@ func TestCheckStorageForNode(t *testing.T) { func TestOnK8sNode(t *testing.T) { ns := "rook-ceph" + opns := "operator" ctx := context.TODO() cephCluster := fakeCluster(ns) objects := []runtime.Object{ @@ -149,24 +150,25 @@ func TestOnK8sNode(t *testing.T) { cephCluster.Status.Phase = k8sutil.ReadyStatus client = getFakeClient(objects...) clientCluster.client = client - b := clientCluster.onK8sNode(ctx, node) + b := clientCluster.onK8sNode(ctx, node, opns) assert.True(t, b) // node will not reconcile - b = clientCluster.onK8sNode(ctx, node) + b = clientCluster.onK8sNode(ctx, node, opns) assert.False(t, b) } func TestHandleNodeFailure(t *testing.T) { - ns := "rook-ceph" + clusterns := "rook-ceph" + opns := "operator" ctx := context.TODO() - cephCluster := fakeCluster(ns) + cephCluster := fakeCluster(clusterns) objects := []runtime.Object{ cephCluster, } executor := &exectest.MockExecutor{} client := getFakeClient(objects...) - c := newClientCluster(client, ns, &clusterd.Context{ + c := newClientCluster(client, clusterns, &clusterd.Context{ Executor: executor, Clientset: k8sFake.NewSimpleClientset(), ApiExtensionsClient: apifake.NewSimpleClientset(), @@ -179,7 +181,7 @@ func TestHandleNodeFailure(t *testing.T) { case command == "ceph" && args[0] == "status": return `{"entity":[{"addr": [{"addr": "10.244.0.12:0", "nonce":3247243972}]}], "client_metadata":{"root":"/"}}`, nil case command == "ceph" && args[0] == "tell": - return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca"}}]`, nil + return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca","hostname":"fakenode"}}]`, nil } return "", errors.Errorf("unexpected rbd/ceph command %q", args) @@ -205,8 +207,8 @@ func TestHandleNodeFailure(t *testing.T) { }, }, VolumesInUse: []corev1.UniqueVolumeName{ - "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", - "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", + "kubernetes.io/csi/operator.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", + "kubernetes.io/csi/operator.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", }, }, } @@ -215,15 +217,15 @@ func TestHandleNodeFailure(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841ca", Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", ns), + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns), "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, }, }, Spec: corev1.PersistentVolumeSpec{ PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", ns), + Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns), VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", VolumeAttributes: map[string]string{ "pool": "replicapool", @@ -238,15 +240,15 @@ func TestHandleNodeFailure(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842ca", Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, }, }, Spec: corev1.PersistentVolumeSpec{ PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), + Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", VolumeAttributes: map[string]string{ "fsName": "myfs", @@ -262,15 +264,15 @@ func TestHandleNodeFailure(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb", Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", ns), + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns), "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, }, }, Spec: corev1.PersistentVolumeSpec{ PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", ns), + Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns), VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", VolumeAttributes: map[string]string{}, }, @@ -282,15 +284,15 @@ func TestHandleNodeFailure(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842cb", Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, }, }, Spec: corev1.PersistentVolumeSpec{ PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", ns), + Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", VolumeAttributes: map[string]string{ "staticVolume": "true", @@ -304,15 +306,15 @@ func TestHandleNodeFailure(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cc", Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.csi.rbd.com", ns), + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.csi.rbd.com", opns), "volume.kubernetes.io/provisioner-deletion-secret-name": "csi-rbd-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, }, }, Spec: corev1.PersistentVolumeSpec{ PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.rbd.csi.com", ns), + Driver: fmt.Sprintf("%s.rbd.csi.com", opns), VolumeHandle: "0001-0009-csi-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", VolumeAttributes: map[string]string{ "pool": "replicapool", @@ -332,12 +334,12 @@ func TestHandleNodeFailure(t *testing.T) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "rook-ceph-mon", - Namespace: ns, + Namespace: clusterns, }, Data: secrets, Type: k8sutil.RookType, } - _, err := c.context.Clientset.CoreV1().Secrets(ns).Create(ctx, secret, metav1.CreateOptions{}) + _, err := c.context.Clientset.CoreV1().Secrets(clusterns).Create(ctx, secret, metav1.CreateOptions{}) assert.NoError(t, err) _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, rbdPV, metav1.CreateOptions{}) @@ -350,15 +352,15 @@ func TestHandleNodeFailure(t *testing.T) { assert.NoError(t, err) // When out-of-service taint is added - err = c.handleNodeFailure(ctx, cephCluster, node) + err = c.handleNodeFailure(ctx, cephCluster, node, opns) assert.NoError(t, err) networkFenceRbd := &addonsv1alpha1.NetworkFence{} - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns)}, networkFenceRbd) + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns)}, networkFenceRbd) assert.NoError(t, err) networkFenceCephFs := &addonsv1alpha1.NetworkFence{} - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, ns)}, networkFenceCephFs) + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns)}, networkFenceCephFs) assert.NoError(t, err) networkFences := &addonsv1alpha1.NetworkFenceList{} @@ -385,11 +387,11 @@ func TestHandleNodeFailure(t *testing.T) { pvList, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) assert.NoError(t, err) - rbdVolumesInUse, _ := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) - rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse) + rbdVolumesInUse, _ := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns) + rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns) assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `rbdPV` - err = c.handleNodeFailure(ctx, cephCluster, node) + err = c.handleNodeFailure(ctx, cephCluster, node, opns) assert.NoError(t, err) // For static cephfs pv @@ -399,15 +401,15 @@ func TestHandleNodeFailure(t *testing.T) { pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) assert.NoError(t, err) - _, cephFSVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) + _, cephFSVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns) cephFSVolumesInUseMap := make(map[string]struct{}) for _, vol := range cephFSVolumesInUse { cephFSVolumesInUseMap[vol] = struct{}{} } - cephFSPVList := listRWOCephFSPV(pvList, cephCluster, cephFSVolumesInUseMap) + cephFSPVList := listRWOCephFSPV(pvList, cephCluster, cephFSVolumesInUseMap, opns) assert.Equal(t, len(cephFSPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `cephfsPV` - err = c.handleNodeFailure(ctx, cephCluster, node) + err = c.handleNodeFailure(ctx, cephCluster, node, opns) assert.NoError(t, err) // For pv not provisioned by CSI @@ -417,23 +419,30 @@ func TestHandleNodeFailure(t *testing.T) { pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) assert.NoError(t, err) - rbdVolumesInUse, _ = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) - rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse) + rbdVolumesInUse, _ = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns) + rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns) assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `PV` - err = c.handleNodeFailure(ctx, cephCluster, node) + err = c.handleNodeFailure(ctx, cephCluster, node, opns) assert.NoError(t, err) // When out-of-service taint is removed node.Spec.Taints = []corev1.Taint{} - - err = c.handleNodeFailure(ctx, cephCluster, node) + networkFenceRbd.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage + err = c.client.Update(ctx, networkFenceRbd) assert.NoError(t, err) - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns), Namespace: cephCluster.Namespace}, networkFenceRbd) + networkFenceCephFs.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage + err = c.client.Update(ctx, networkFenceCephFs) + assert.NoError(t, err) + + err = c.handleNodeFailure(ctx, cephCluster, node, opns) + assert.NoError(t, err) + + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceRbd) assert.Error(t, err, kerrors.IsNotFound(err)) - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, ns), Namespace: cephCluster.Namespace}, networkFenceCephFs) + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceCephFs) assert.Error(t, err, kerrors.IsNotFound(err)) } @@ -463,7 +472,7 @@ func TestGetCephVolumesInUse(t *testing.T) { assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com") assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004") - trimRbdVolInUse, trimCephFSVolInUse := getCephVolumesInUse(cephCluster, volInUse) + trimRbdVolInUse, trimCephFSVolInUse := getCephVolumesInUse(cephCluster, volInUse, cephCluster.Namespace) expectedRbd := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003"} expectedCephfs := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004"} diff --git a/pkg/operator/ceph/cr_manager.go b/pkg/operator/ceph/cr_manager.go index 6805749be..0652b1663 100644 --- a/pkg/operator/ceph/cr_manager.go +++ b/pkg/operator/ceph/cr_manager.go @@ -102,13 +102,13 @@ var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Conte // AddToManager adds all the registered controllers to the passed manager. // each controller package will have an Add method listed in AddToManagerFuncs // which will setup all the necessary watch -func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context) error { +func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context, opconfig opcontroller.OperatorConfig) error { if c == nil { return errors.New("nil context passed") } // Run CephCluster CR - if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext); err != nil { + if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext, opconfig); err != nil { return err } @@ -173,7 +173,7 @@ func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan erro } // Add the registered controllers to the manager (entrypoint for controllers) - err = o.addToManager(mgr, controllerOpts, context) + err = o.addToManager(mgr, controllerOpts, context, *o.config) if err != nil { mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager") return