diff --git a/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md b/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md index 95b54bca6..131ce5367 100644 --- a/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md @@ -196,7 +196,7 @@ The erasure coded pool must be set as the `dataPool` parameter in ## Node Loss -If a node goes down where a pod is running where a RBD RWO volume is mounted, the volume cannot automatically be mounted on another node. The node must be guaranteed to be offline before the volume can be mounted on another node. +If a node goes down where a pod is running where a volume is mounted, the volume cannot automatically be mounted on another node. The node must be guaranteed to be offline before the volume can be mounted on another node. ### Handling Node Loss @@ -209,7 +209,7 @@ kubectl taint nodes node.kubernetes.io/out-of-service=nodeshutdown:N kubectl taint nodes node.kubernetes.io/out-of-service=nodeshutdown:NoSchedule ``` -After the taint is added to the node, the CephCSI driver will automatically handle the network fencing to prevent connections to Ceph from the RBD volume on that node. +After the taint is added to the node, the CephCSI driver will automatically handle the network fencing to prevent connections to Ceph from the volume on that node. ### Node Recovery diff --git a/pkg/operator/ceph/cluster/watcher_test.go b/pkg/operator/ceph/cluster/watcher_test.go index 7e72dd495..c523c7e4c 100644 --- a/pkg/operator/ceph/cluster/watcher_test.go +++ b/pkg/operator/ceph/cluster/watcher_test.go @@ -152,349 +152,6 @@ func TestOnK8sNode(t *testing.T) { assert.False(t, b) } -<<<<<<< HEAD -func TestHandleNodeFailure(t *testing.T) { - clusterns := "rook-ceph" - opns := "operator" - ctx := context.TODO() - cephCluster := fakeCluster(clusterns) - objects := []runtime.Object{ - cephCluster, - } - executor := &exectest.MockExecutor{} - client := getFakeClient(objects...) - c := newClientCluster(client, clusterns, &clusterd.Context{ - Executor: executor, - Clientset: k8sFake.NewSimpleClientset(), - ApiExtensionsClient: apifake.NewSimpleClientset(), - }) - - executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { - switch { - case command == "rbd" && args[0] == "status": - return `{"watchers":[{"address":"192.168.39.137:0/3762982934","client":4307,"cookie":18446462598732840961}]}`, nil - case command == "ceph" && args[0] == "status": - return `{"entity":[{"addr": [{"addr": "10.244.0.12:0", "nonce":3247243972}]}], "client_metadata":{"root":"/"}}`, nil - case command == "ceph" && args[0] == "tell": - return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca","hostname":"fakenode"}}]`, nil - - } - return "", errors.Errorf("unexpected rbd/ceph command %q", args) - } - - node := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fakenode", - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: corev1.TaintNodeOutOfService, - Value: string(corev1.TaintEffectNoSchedule), - }, - }, - }, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - { - Status: corev1.ConditionStatus(corev1.ConditionTrue), - Type: corev1.NodeConditionType(corev1.NodeReady), - }, - }, - VolumesInUse: []corev1.UniqueVolumeName{ - "kubernetes.io/csi/operator.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", - "kubernetes.io/csi/operator.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", - }, - }, - } - - rbdPV := &corev1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841ca", - Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns), - "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, - }, - }, - Spec: corev1.PersistentVolumeSpec{ - PersistentVolumeSource: corev1.PersistentVolumeSource{ - CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns), - VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", - VolumeAttributes: map[string]string{ - "pool": "replicapool", - "imageName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b841ca", - }, - }, - }, - }, - } - - cephfsPV := &corev1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842ca", - Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), - "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, - }, - }, - Spec: corev1.PersistentVolumeSpec{ - PersistentVolumeSource: corev1.PersistentVolumeSource{ - CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), - VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", - VolumeAttributes: map[string]string{ - "fsName": "myfs", - "subvolumePath": "/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", - "subvolumeName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca", - }, - }, - }, - }, - } - - staticRbdPV := &corev1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb", - Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns), - "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, - }, - }, - Spec: corev1.PersistentVolumeSpec{ - PersistentVolumeSource: corev1.PersistentVolumeSource{ - CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns), - VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", - VolumeAttributes: map[string]string{}, - }, - }, - }, - } - - staticCephfsPV := &corev1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842cb", - Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), - "volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, - }, - }, - Spec: corev1.PersistentVolumeSpec{ - PersistentVolumeSource: corev1.PersistentVolumeSource{ - CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns), - VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", - VolumeAttributes: map[string]string{ - "staticVolume": "true", - }, - }, - }, - }, - } - - pvNotProvisionByCSI := &corev1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cc", - Annotations: map[string]string{ - "pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.csi.rbd.com", opns), - "volume.kubernetes.io/provisioner-deletion-secret-name": "csi-rbd-provisioner", - "volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns, - }, - }, - Spec: corev1.PersistentVolumeSpec{ - PersistentVolumeSource: corev1.PersistentVolumeSource{ - CSI: &corev1.CSIPersistentVolumeSource{ - Driver: fmt.Sprintf("%s.rbd.csi.com", opns), - VolumeHandle: "0001-0009-csi-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", - VolumeAttributes: map[string]string{ - "pool": "replicapool", - "imageName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b841ca", - }, - }, - }, - }, - } - - // Mock clusterInfo - secrets := map[string][]byte{ - "fsid": []byte("c47cac40-9bee-4d52-823b-ccd803ba5bfe"), - "mon-secret": []byte("monsecret"), - "admin-secret": []byte("adminsecret"), - } - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rook-ceph-mon", - Namespace: clusterns, - }, - Data: secrets, - Type: k8sutil.RookType, - } - _, err := c.context.Clientset.CoreV1().Secrets(clusterns).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) - - _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, rbdPV, metav1.CreateOptions{}) - assert.NoError(t, err) - - _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, cephfsPV, metav1.CreateOptions{}) - assert.NoError(t, err) - - _, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &v1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "networkfences.csiaddons.openshift.io"}}, metav1.CreateOptions{}) - assert.NoError(t, err) - - // When out-of-service taint is added - err = c.handleNodeFailure(ctx, cephCluster, node, opns) - assert.NoError(t, err) - - networkFenceRbd := &addonsv1alpha1.NetworkFence{} - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns)}, networkFenceRbd) - assert.NoError(t, err) - - networkFenceCephFs := &addonsv1alpha1.NetworkFence{} - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns)}, networkFenceCephFs) - assert.NoError(t, err) - - networkFences := &addonsv1alpha1.NetworkFenceList{} - err = c.client.List(ctx, networkFences) - assert.NoError(t, err) - var rbdCount, cephFsCount int - - for _, fence := range networkFences.Items { - // Check if the resource is in the desired namespace - if strings.Contains(fence.Name, rbdDriver) { - rbdCount++ - } else if strings.Contains(fence.Name, cephfsDriver) { - cephFsCount++ - } - } - - assert.Equal(t, 1, rbdCount) - assert.Equal(t, 1, cephFsCount) - - // For static rbd pv - _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticRbdPV, metav1.CreateOptions{}) - assert.NoError(t, err) - - pvList, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) - assert.NoError(t, err) - - rbdVolumesInUse, _ := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns) - rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns) - assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `rbdPV` - - err = c.handleNodeFailure(ctx, cephCluster, node, opns) - assert.NoError(t, err) - - // For static cephfs pv - _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticCephfsPV, metav1.CreateOptions{}) - assert.NoError(t, err) - - pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) - assert.NoError(t, err) - - _, cephFSVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns) - cephFSVolumesInUseMap := make(map[string]struct{}) - for _, vol := range cephFSVolumesInUse { - cephFSVolumesInUseMap[vol] = struct{}{} - } - cephFSPVList := listRWOCephFSPV(pvList, cephCluster, cephFSVolumesInUseMap, opns) - assert.Equal(t, len(cephFSPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `cephfsPV` - - err = c.handleNodeFailure(ctx, cephCluster, node, opns) - assert.NoError(t, err) - - // For pv not provisioned by CSI - _, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, pvNotProvisionByCSI, metav1.CreateOptions{}) - assert.NoError(t, err) - - pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) - assert.NoError(t, err) - - rbdVolumesInUse, _ = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns) - rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns) - assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `PV` - - err = c.handleNodeFailure(ctx, cephCluster, node, opns) - assert.NoError(t, err) - - // When out-of-service taint is removed - node.Spec.Taints = []corev1.Taint{} - networkFenceRbd.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage - err = c.client.Update(ctx, networkFenceRbd) - assert.NoError(t, err) - - networkFenceCephFs.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage - err = c.client.Update(ctx, networkFenceCephFs) - assert.NoError(t, err) - - err = c.handleNodeFailure(ctx, cephCluster, node, opns) - assert.NoError(t, err) - - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceRbd) - assert.Error(t, err, kerrors.IsNotFound(err)) - - err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceCephFs) - assert.Error(t, err, kerrors.IsNotFound(err)) -} - -func TestGetCephVolumesInUse(t *testing.T) { - cephCluster := fakeCluster("rook-ceph") - volInUse := []corev1.UniqueVolumeName{ - "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", - "kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003", - "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", - "kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004", - } - - splitVolInUse := trimeVolumeInUse(volInUse[0]) - assert.Equal(t, splitVolInUse[0], "rook-ceph.rbd.csi.ceph.com") - assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002") - - splitVolInUse = trimeVolumeInUse(volInUse[1]) - assert.Equal(t, splitVolInUse[0], "rook-ceph.rbd.csi.ceph.com") - assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003") - - splitVolInUse = trimeVolumeInUse(volInUse[2]) - assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com") - assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001") - - splitVolInUse = trimeVolumeInUse(volInUse[3]) - assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com") - assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004") - - trimRbdVolInUse, trimCephFSVolInUse := getCephVolumesInUse(cephCluster, volInUse, cephCluster.Namespace) - - expectedRbd := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003"} - expectedCephfs := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004"} - - assert.Equal(t, expectedRbd, trimRbdVolInUse) - assert.Equal(t, expectedCephfs, trimCephFSVolInUse) -} - -func TestRBDStatusUnMarshal(t *testing.T) { - output := `{"watchers":[{"address":"192.168.39.137:0/3762982934","client":4307,"cookie":18446462598732840961}]}` - - listIP, err := rbdStatusUnMarshal([]byte(output)) - assert.NoError(t, err) - assert.Equal(t, listIP[0], "192.168.39.137/32") -} - -func TestConcatenateWatcherIp(t *testing.T) { - WatcherIP := concatenateWatcherIp("192.168.39.137:0/3762982934") - assert.Equal(t, WatcherIP, "192.168.39.137/32") -} - -func TestFenceResourceName(t *testing.T) { - FenceName := fenceResourceName("fakenode", "rbd", "rook-ceph") - assert.Equal(t, FenceName, "fakenode-rbd-rook-ceph") -} - -======= ->>>>>>> 20f350a71 (csi: remove automated node fencing code) func TestOnDeviceCMUpdate(t *testing.T) { // Set DEBUG logging capnslog.SetGlobalLogLevel(capnslog.DEBUG)