diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 6466607ea..4638f2bab 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -99,16 +99,17 @@ type ReconcileCephCluster struct { context *clusterd.Context clusterController *ClusterController opManagerContext context.Context + opConfig opcontroller.OperatorConfig } // Add creates a new CephCluster Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) error { - return add(opManagerContext, mgr, newReconciler(mgr, ctx, clusterController, opManagerContext), ctx) +func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(opManagerContext, mgr, newReconciler(mgr, ctx, clusterController, opManagerContext, opConfig), ctx, opConfig) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) reconcile.Reconciler { +func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { // add "rook-" prefix to the controller name to make sure it is clear to all reading the events // that they are coming from Rook. The controller name already has context that it is for Ceph // and from the cluster controller. @@ -118,12 +119,13 @@ func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController client: mgr.GetClient(), scheme: mgr.GetScheme(), context: ctx, + opConfig: opConfig, clusterController: clusterController, opManagerContext: opManagerContext, } } -func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context) error { +func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context, opConfig opcontroller.OperatorConfig) error { // Create a new controller c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r}) if err != nil { @@ -170,7 +172,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco mgr.GetCache(), &corev1.Node{TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: corev1.SchemeGroupVersion.String()}}, handler.EnqueueRequestsFromMapFunc(handlerFunc), - predicateForNodeWatcher(opManagerContext, mgr.GetClient(), context)) + predicateForNodeWatcher(opManagerContext, mgr.GetClient(), context, opConfig.OperatorNamespace)) err = c.Watch(nodeKind) if err != nil { return err diff --git a/pkg/operator/ceph/cluster/predicate.go b/pkg/operator/ceph/cluster/predicate.go index f7f5112a6..709b5b6bf 100644 --- a/pkg/operator/ceph/cluster/predicate.go +++ b/pkg/operator/ceph/cluster/predicate.go @@ -52,11 +52,11 @@ func shouldReconcileChangedNode(objOld, objNew *corev1.Node) bool { } // predicateForNodeWatcher is the predicate function to trigger reconcile on Node events -func predicateForNodeWatcher(ctx context.Context, client client.Client, context *clusterd.Context) predicate.Funcs { +func predicateForNodeWatcher(ctx context.Context, client client.Client, context *clusterd.Context, opNamespace string) predicate.Funcs { return predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { clientCluster := newClientCluster(client, e.Object.GetNamespace(), context) - return clientCluster.onK8sNode(ctx, e.Object) + return clientCluster.onK8sNode(ctx, e.Object, opNamespace) }, UpdateFunc: func(e event.UpdateEvent) bool { @@ -67,7 +67,7 @@ func predicateForNodeWatcher(ctx context.Context, client client.Client, context } clientCluster := newClientCluster(client, e.ObjectNew.GetNamespace(), context) - return clientCluster.onK8sNode(ctx, e.ObjectNew) + return clientCluster.onK8sNode(ctx, e.ObjectNew, opNamespace) } } return false diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index a79c71221..f57021dde 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -60,7 +60,8 @@ var ( // drivers that supports fencing, used in naming networkFence object const ( - rbdDriver = "rbd" + rbdDriver = "rbd" + cephfsDriver = "cephfs" ) func newClientCluster(client client.Client, namespace string, context *clusterd.Context) *clientCluster { @@ -80,7 +81,7 @@ func checkStorageForNode(cluster *cephv1.CephCluster) bool { } // onK8sNode is triggered when a node is added in the Kubernetes cluster -func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bool { +func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, opNamespace string) bool { node, ok := object.(*corev1.Node) if !ok { return false @@ -90,7 +91,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bo cluster := c.getCephCluster() // Continue reconcile in case of failure too since we don't want to block other node reconcile - if err := c.handleNodeFailure(ctx, cluster, node); err != nil { + if err := c.handleNodeFailure(ctx, cluster, node, opNamespace); err != nil { logger.Errorf("failed to handle node failure. %v", err) } @@ -157,7 +158,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bo return false } -func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node) error { +func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node, opNamespace string) error { watchForNodeLoss, err := k8sutil.GetOperatorSetting(ctx, c.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_WATCH_FOR_NODE_FAILURE", "true") if err != nil { return pkgerror.Wrapf(err, "failed to get configmap value `ROOK_WATCH_FOR_NODE_FAILURE`.") @@ -198,7 +199,7 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C } if nodeHasOutOfServiceTaint { - err := c.fenceNode(ctx, node, cluster) + err := c.fenceNode(ctx, node, cluster, opNamespace) if err != nil { return pkgerror.Wrapf(err, "failed to create network fence for node %q.", node.Name) } @@ -210,10 +211,14 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C return pkgerror.Wrapf(err, "failed to delete rbd network fence for node %q.", node.Name) } + err = c.unfenceAndDeleteNetworkFence(ctx, *node, cluster, cephfsDriver) + if err != nil { + return pkgerror.Wrapf(err, "failed to delete cephFS network fence for node %q.", node.Name) + } return nil } -func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster) error { +func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster, opNamespace string) error { volumesInuse := node.Status.VolumesInUse if len(volumesInuse) == 0 { logger.Debugf("no volumes in use for node %q", node.Name) @@ -221,9 +226,9 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste } logger.Debugf("volumesInuse %s", volumesInuse) - rbdVolumesInUse := getCephVolumesInUse(cluster, volumesInuse) - if len(rbdVolumesInUse) == 0 { - logger.Debugf("no rbd volumes in use for out of service node %q", node.Name) + rbdVolumesInUse, cephFSVolumeInUse := getCephVolumesInUse(cluster, volumesInuse, opNamespace) + if len(rbdVolumesInUse) == 0 && len(cephFSVolumeInUse) == 0 { + logger.Debugf("no rbd or cephfs subvolumes in use for out of service node %q", node.Name) return nil } @@ -233,7 +238,7 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste } if len(rbdVolumesInUse) != 0 { - rbdPVList := listRBDPV(listPVs, cluster, rbdVolumesInUse) + rbdPVList := listRBDPV(listPVs, cluster, rbdVolumesInUse, opNamespace) if len(rbdPVList) == 0 { logger.Debug("No rbd PVs found on the node") } else { @@ -262,22 +267,59 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste } } + if len(cephFSVolumeInUse) != 0 { + cephFSVolumeInUseMap := make(map[string]struct{}) + for _, vol := range cephFSVolumeInUse { + cephFSVolumeInUseMap[vol] = struct{}{} + } + cephFSPVList := listRWOCephFSPV(listPVs, cluster, cephFSVolumeInUseMap, opNamespace) + if len(cephFSPVList) == 0 { + logger.Debug("No cephfs PVs found on the node %s", node.Name) + return nil + } + logger.Infof("node %q require fencing, found cephfs subvolumes in use", node.Name) + clusterInfo, _, _, err := opcontroller.LoadClusterInfo(c.context, ctx, cluster.Namespace, &cluster.Spec) + if err != nil { + return pkgerror.Wrap(err, "Failed to load cluster info.") + } + + for i := range cephFSPVList { + err = c.fenceCephFSSubvolume(ctx, node, cluster, clusterInfo, cephFSPVList[i]) + // We only need to create the network fence for any one of cephFS pv. + if err == nil { + break + } + + // continue to fence next cephfs subvolume if active client not found + if stderrors.Is(err, errActiveClientNotFound) { + continue + } + if i == len(cephFSPVList)-1 { + return pkgerror.Wrap(err, "failed to fence cephfs subvolumes") + } + logger.Errorf("failed to fence cephfs subvolume %q, trying next cephfs subvolume", cephFSPVList[i].Name) + } + + } return nil } -func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) []string { - var rbdVolumesInUse []string +func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName, opNamespace string) ([]string, []string) { + var rbdVolumesInUse, cephFSVolumeInUse []string for _, volume := range volumesInUse { splitVolumeInUseBased := trimeVolumeInUse(volume) logger.Infof("volumeInUse after split based on '^' %v", splitVolumeInUseBased) - if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) { + if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", opNamespace) { rbdVolumesInUse = append(rbdVolumesInUse, splitVolumeInUseBased[1]) } - } - return rbdVolumesInUse + if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.cephfs.csi.ceph.com", opNamespace) { + cephFSVolumeInUse = append(cephFSVolumeInUse, splitVolumeInUseBased[1]) + } + } + return rbdVolumesInUse, cephFSVolumeInUse } func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string { @@ -286,7 +328,7 @@ func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string { return splitVolumeInUseBased } -func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, rbdVolumesInUse []string) []corev1.PersistentVolume { +func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, rbdVolumesInUse []string, opNamespace string) []corev1.PersistentVolume { var listRbdPV []corev1.PersistentVolume for _, pv := range listPVs.Items { @@ -296,7 +338,7 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster continue } - if pv.Spec.CSI.Driver == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) { + if pv.Spec.CSI.Driver == fmt.Sprintf("%s.rbd.csi.ceph.com", opNamespace) { // Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes. if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) { continue @@ -316,6 +358,35 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster return listRbdPV } +func listRWOCephFSPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, cephFSVolumesInUse map[string]struct{}, opNamespace string) []corev1.PersistentVolume { + var listCephFSPV []corev1.PersistentVolume + + for _, pv := range listPVs.Items { + // Skip if pv is not provisioned by CSI + if pv.Spec.CSI == nil { + logger.Debugf("PV %q is not provisioned by CSI", pv.Name) + continue + } + + if pv.Spec.CSI.Driver == fmt.Sprintf("%s.cephfs.csi.ceph.com", opNamespace) { + // Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes. + if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) { + continue + } + + if pv.Spec.CSI.VolumeAttributes["staticVolume"] == "true" { + logger.Debugf("skipping, static PV %q", pv.Name) + continue + } + // Check if the volume is in use + if _, exists := cephFSVolumesInUse[pv.Spec.CSI.VolumeHandle]; exists { + listCephFSPV = append(listCephFSPV, pv) + } + } + } + return listCephFSPV +} + // pvSupportsMultiNodeAccess returns true if the PV access modes contain ReadWriteMany or ReadOnlyMany. func pvSupportsMultiNodeAccess(accessModes []corev1.PersistentVolumeAccessMode) bool { for _, accessMode := range accessModes { @@ -358,6 +429,89 @@ func (c *clientCluster) fenceRbdImage( return nil } +func (c *clientCluster) fenceCephFSSubvolume( + ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster, + clusterInfo *cephclient.ClusterInfo, cephFSPV corev1.PersistentVolume) error { + + logger.Infof("fencing cephfs subvolume %q on node %q", cephFSPV.Name, node.Name) + + status, err := cephclient.StatusWithUser(c.context, clusterInfo) + if err != nil { + return pkgerror.Wrap(err, "failed to get ceph status for check active mds") + } + + var activeMDS string + for _, fsRank := range status.Fsmap.ByRank { + if fsRank.Status == "up:active" { + activeMDS = fsRank.Name + } + } + + args := []string{"tell", fmt.Sprintf("mds.%s", activeMDS), "client", "ls", "--format", "json"} + cmd := cephclient.NewCephCommand(c.context, clusterInfo, args) + cmd.JsonOutput = true + + buf, err := cmd.Run() + if err != nil { + return fmt.Errorf("failed to list watchers for cephfs subvolumeName %s. %v", cephFSPV.Spec.CSI.VolumeAttributes["subvolumeName"], err) + } + ips, err := cephFSMDSClientMarshal(buf, node.Name, cephFSPV) + if err != nil { + return pkgerror.Wrap(err, "failed to unmarshal cephfs mds output") + } + + if len(ips) == 0 { + logger.Infof("no active mds clients found for cephfs subvolume %q", cephFSPV.Name) + return errActiveClientNotFound + } + + err = c.createNetworkFence(ctx, cephFSPV, node, cluster, ips, cephfsDriver) + if err != nil { + return pkgerror.Wrapf(err, "failed to create network fence for node %q", node.Name) + } + + return nil +} + +func cephFSMDSClientMarshal(output []byte, nodeName string, cephFSPV corev1.PersistentVolume) ([]string, error) { + type entity struct { + Addr struct { + Addr string `json:"addr"` + Nonce int `json:"nonce"` + } `json:"addr"` + } + + type clientMetadata struct { + Root string `json:"root"` + Hostname string `json:"hostname"` + } + + type cephFSData struct { + Entity entity `json:"entity"` + ClientMetadata clientMetadata `json:"client_metadata"` + } + + var data []cephFSData + err := json.Unmarshal([]byte(output), &data) + if err != nil { + return []string{}, pkgerror.Wrapf(err, "failed to unmarshal cephFS data output") + } + + watcherIPlist := []string{} + for _, d := range data { + if d.ClientMetadata.Hostname != "" { + if strings.Contains(nodeName, d.ClientMetadata.Hostname) && cephFSPV.Spec.CSI.VolumeAttributes["subvolumePath"] == d.ClientMetadata.Root { + logger.Infof("cephfs mds client ips to fence %v", d.Entity.Addr) + watcherIP := concatenateWatcherIp(d.Entity.Addr.Addr) + watcherIPlist = append(watcherIPlist, watcherIP) + break + } + } + } + + return watcherIPlist, nil +} + func rbdStatusUnMarshal(output []byte) ([]string, error) { type rbdStatus struct { Watchers []struct { @@ -380,7 +534,8 @@ func rbdStatusUnMarshal(output []byte) ([]string, error) { } func concatenateWatcherIp(address string) string { - // address is in format `10.63.0.5:0/1254753579` for rbd + // address is in format `10.63.0.5:0/1254753579` for rbd and + // in the format '10.244.0.12:0' for cephfs // split with separation ':0' to remove nounce and concatenating `/32` to define a network with only one IP address watcherIP := strings.Split(address, ":0")[0] + "/32" return watcherIP diff --git a/pkg/operator/ceph/cluster/watcher_test.go b/pkg/operator/ceph/cluster/watcher_test.go index e3585ae81..f4b83ca07 100644 --- a/pkg/operator/ceph/cluster/watcher_test.go +++ b/pkg/operator/ceph/cluster/watcher_test.go @@ -96,6 +96,7 @@ func TestCheckStorageForNode(t *testing.T) { func TestOnK8sNode(t *testing.T) { ns := "rook-ceph" + opns := "operator" ctx := context.TODO() cephCluster := fakeCluster(ns) objects := []runtime.Object{ @@ -149,24 +150,25 @@ func TestOnK8sNode(t *testing.T) { cephCluster.Status.Phase = k8sutil.ReadyStatus client = getFakeClient(objects...) clientCluster.client = client - b := clientCluster.onK8sNode(ctx, node) + b := clientCluster.onK8sNode(ctx, node, opns) assert.True(t, b) // node will not reconcile - b = clientCluster.onK8sNode(ctx, node) + b = clientCluster.onK8sNode(ctx, node, opns) assert.False(t, b) } func TestHandleNodeFailure(t *testing.T) { - ns := "rook-ceph" + clusterns := "rook-ceph" + opns := "operator" ctx := context.TODO() - cephCluster := fakeCluster(ns) + cephCluster := fakeCluster(clusterns) objects := []runtime.Object{ cephCluster, } executor := &exectest.MockExecutor{} client := getFakeClient(objects...) - c := newClientCluster(client, ns, &clusterd.Context{ + c := newClientCluster(client, clusterns, &clusterd.Context{ Executor: executor, Clientset: k8sFake.NewSimpleClientset(), ApiExtensionsClient: apifake.NewSimpleClientset(), @@ -179,7 +181,7 @@ func TestHandleNodeFailure(t *testing.T) { case command == "ceph" && args[0] == "status": return `{"entity":[{"addr": [{"addr": "10.244.0.12:0", "nonce":3247243972}]}], "client_metadata":{"root":"/"}}`, nil case command == "ceph" && args[0] == "tell": - return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca"}}]`, nil + return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca","hostname":"fakenode"}}]`, nil } return "", errors.Errorf("unexpected rbd/ceph command %q", args) @@ -205,7 +207,8 @@ func TestHandleNodeFailure(t *testing.T) { }, }, VolumesInUse: []corev1.UniqueVolumeName{ - "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", + "kubernetes.io/csi/operator.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", + "kubernetes.io/csi/operator.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", }, }, } @@ -214,15 +217,15 @@ func TestHandleNodeFailure(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841ca", Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", ns), + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns), "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, }, }, Spec: corev1.PersistentVolumeSpec{ PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", ns), + Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns), VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", VolumeAttributes: map[string]string{ "pool": "replicapool", @@ -233,19 +236,43 @@ func TestHandleNodeFailure(t *testing.T) { }, } - staticRbdPV := &corev1.PersistentVolume{ + cephfsPV := &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb", + Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842ca", Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", ns), - "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), + "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, }, }, Spec: corev1.PersistentVolumeSpec{ PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", ns), + Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), + VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", + VolumeAttributes: map[string]string{ + "fsName": "myfs", + "subvolumePath": "/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", + "subvolumeName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", + }, + }, + }, + }, + } + + staticRbdPV := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb", + Annotations: map[string]string{ + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns), + "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner", + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns), VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", VolumeAttributes: map[string]string{}, }, @@ -253,19 +280,41 @@ func TestHandleNodeFailure(t *testing.T) { }, } + staticCephfsPV := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842cb", + Annotations: map[string]string{ + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), + "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), + VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", + VolumeAttributes: map[string]string{ + "staticVolume": "true", + }, + }, + }, + }, + } + pvNotProvisionByCSI := &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cc", Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.csi.rbd.com", ns), + "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.csi.rbd.com", opns), "volume.kubernetes.io/provisioner-deletion-secret-name": "csi-rbd-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": ns, + "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, }, }, Spec: corev1.PersistentVolumeSpec{ PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.rbd.csi.com", ns), + Driver: fmt.Sprintf("%s.rbd.csi.com", opns), VolumeHandle: "0001-0009-csi-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", VolumeAttributes: map[string]string{ "pool": "replicapool", @@ -285,41 +334,51 @@ func TestHandleNodeFailure(t *testing.T) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "rook-ceph-mon", - Namespace: ns, + Namespace: clusterns, }, Data: secrets, Type: k8sutil.RookType, } - _, err := c.context.Clientset.CoreV1().Secrets(ns).Create(ctx, secret, metav1.CreateOptions{}) + _, err := c.context.Clientset.CoreV1().Secrets(clusterns).Create(ctx, secret, metav1.CreateOptions{}) assert.NoError(t, err) _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, rbdPV, metav1.CreateOptions{}) assert.NoError(t, err) + _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, cephfsPV, metav1.CreateOptions{}) + assert.NoError(t, err) + _, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &v1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "networkfences.csiaddons.openshift.io"}}, metav1.CreateOptions{}) assert.NoError(t, err) // When out-of-service taint is added - err = c.handleNodeFailure(ctx, cephCluster, node) + err = c.handleNodeFailure(ctx, cephCluster, node, opns) assert.NoError(t, err) networkFenceRbd := &addonsv1alpha1.NetworkFence{} - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns)}, networkFenceRbd) + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns)}, networkFenceRbd) + assert.NoError(t, err) + + networkFenceCephFs := &addonsv1alpha1.NetworkFence{} + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns)}, networkFenceCephFs) assert.NoError(t, err) networkFences := &addonsv1alpha1.NetworkFenceList{} err = c.client.List(ctx, networkFences) assert.NoError(t, err) - var rbdCount int + var rbdCount, cephFsCount int for _, fence := range networkFences.Items { // Check if the resource is in the desired namespace if strings.Contains(fence.Name, rbdDriver) { rbdCount++ + } else if strings.Contains(fence.Name, cephfsDriver) { + cephFsCount++ } } assert.Equal(t, 1, rbdCount) + assert.Equal(t, 1, cephFsCount) // For static rbd pv _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticRbdPV, metav1.CreateOptions{}) @@ -328,14 +387,29 @@ func TestHandleNodeFailure(t *testing.T) { pvList, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) assert.NoError(t, err) - rbdVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) - rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse) + rbdVolumesInUse, _ := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns) + rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns) assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `rbdPV` - err = c.handleNodeFailure(ctx, cephCluster, node) + err = c.handleNodeFailure(ctx, cephCluster, node, opns) assert.NoError(t, err) - err = c.handleNodeFailure(ctx, cephCluster, node) + // For static cephfs pv + _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticCephfsPV, metav1.CreateOptions{}) + assert.NoError(t, err) + + pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + + _, cephFSVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns) + cephFSVolumesInUseMap := make(map[string]struct{}) + for _, vol := range cephFSVolumesInUse { + cephFSVolumesInUseMap[vol] = struct{}{} + } + cephFSPVList := listRWOCephFSPV(pvList, cephCluster, cephFSVolumesInUseMap, opns) + assert.Equal(t, len(cephFSPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `cephfsPV` + + err = c.handleNodeFailure(ctx, cephCluster, node, opns) assert.NoError(t, err) // For pv not provisioned by CSI @@ -345,25 +419,32 @@ func TestHandleNodeFailure(t *testing.T) { pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) assert.NoError(t, err) - rbdVolumesInUse = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse) - rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse) + rbdVolumesInUse, _ = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns) + rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns) assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `PV` - err = c.handleNodeFailure(ctx, cephCluster, node) + err = c.handleNodeFailure(ctx, cephCluster, node, opns) assert.NoError(t, err) // When out-of-service taint is removed node.Spec.Taints = []corev1.Taint{} - networkFenceRbd.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage err = c.client.Update(ctx, networkFenceRbd) assert.NoError(t, err) - err = c.handleNodeFailure(ctx, cephCluster, node) + networkFenceCephFs.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage + err = c.client.Update(ctx, networkFenceCephFs) assert.NoError(t, err) - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns), Namespace: cephCluster.Namespace}, networkFenceRbd) + err = c.handleNodeFailure(ctx, cephCluster, node, opns) + assert.NoError(t, err) + + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceRbd) assert.Error(t, err, kerrors.IsNotFound(err)) + + err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceCephFs) + assert.Error(t, err, kerrors.IsNotFound(err)) + } func TestGetCephVolumesInUse(t *testing.T) { @@ -371,6 +452,8 @@ func TestGetCephVolumesInUse(t *testing.T) { volInUse := []corev1.UniqueVolumeName{ "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003", + "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", + "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004", } splitVolInUse := trimeVolumeInUse(volInUse[0]) @@ -381,11 +464,21 @@ func TestGetCephVolumesInUse(t *testing.T) { assert.Equal(t, splitVolInUse[0], "rook-ceph.rbd.csi.ceph.com") assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003") - trimRbdVolInUse := getCephVolumesInUse(cephCluster, volInUse) + splitVolInUse = trimeVolumeInUse(volInUse[2]) + assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com") + assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001") + + splitVolInUse = trimeVolumeInUse(volInUse[3]) + assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com") + assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004") + + trimRbdVolInUse, trimCephFSVolInUse := getCephVolumesInUse(cephCluster, volInUse, cephCluster.Namespace) expectedRbd := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003"} + expectedCephfs := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004"} assert.Equal(t, expectedRbd, trimRbdVolInUse) + assert.Equal(t, expectedCephfs, trimCephFSVolInUse) } func TestRBDStatusUnMarshal(t *testing.T) { diff --git a/pkg/operator/ceph/cr_manager.go b/pkg/operator/ceph/cr_manager.go index 6805749be..0652b1663 100644 --- a/pkg/operator/ceph/cr_manager.go +++ b/pkg/operator/ceph/cr_manager.go @@ -102,13 +102,13 @@ var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Conte // AddToManager adds all the registered controllers to the passed manager. // each controller package will have an Add method listed in AddToManagerFuncs // which will setup all the necessary watch -func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context) error { +func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context, opconfig opcontroller.OperatorConfig) error { if c == nil { return errors.New("nil context passed") } // Run CephCluster CR - if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext); err != nil { + if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext, opconfig); err != nil { return err } @@ -173,7 +173,7 @@ func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan erro } // Add the registered controllers to the manager (entrypoint for controllers) - err = o.addToManager(mgr, controllerOpts, context) + err = o.addToManager(mgr, controllerOpts, context, *o.config) if err != nil { mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager") return