Merge pull request #14258 from subhamkrai/add-cephfs-fencing

mds: block of active mds ip only
This commit is contained in:
Travis Nielsen authored and GitHub committed 2024-07-17 11:48:01 -06:00
commit 3ccdb53cb6
5 files changed
+314 -64

No files matched your search

+7 -5
View File
@@ -99,16 +99,17 @@ type ReconcileCephCluster struct {
context *clusterd.Context
clusterController *ClusterController
opManagerContext context.Context
opConfig opcontroller.OperatorConfig
}
// Add creates a new CephCluster Controller and adds it to the Manager. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) error {
return add(opManagerContext, mgr, newReconciler(mgr, ctx, clusterController, opManagerContext), ctx)
func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
return add(opManagerContext, mgr, newReconciler(mgr, ctx, clusterController, opManagerContext, opConfig), ctx, opConfig)
}
// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) reconcile.Reconciler {
func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
// add "rook-" prefix to the controller name to make sure it is clear to all reading the events
// that they are coming from Rook. The controller name already has context that it is for Ceph
// and from the cluster controller.
@@ -118,12 +119,13 @@ func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
context: ctx,
opConfig: opConfig,
clusterController: clusterController,
opManagerContext: opManagerContext,
}
}
func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context) error {
func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context, opConfig opcontroller.OperatorConfig) error {
// Create a new controller
c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
if err != nil {
@@ -170,7 +172,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco
mgr.GetCache(),
&corev1.Node{TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: corev1.SchemeGroupVersion.String()}},
handler.EnqueueRequestsFromMapFunc(handlerFunc),
predicateForNodeWatcher(opManagerContext, mgr.GetClient(), context))
predicateForNodeWatcher(opManagerContext, mgr.GetClient(), context, opConfig.OperatorNamespace))
err = c.Watch(nodeKind)
if err != nil {
return err
+3 -3
View File
@@ -52,11 +52,11 @@ func shouldReconcileChangedNode(objOld, objNew *corev1.Node) bool {
}
// predicateForNodeWatcher is the predicate function to trigger reconcile on Node events
func predicateForNodeWatcher(ctx context.Context, client client.Client, context *clusterd.Context) predicate.Funcs {
func predicateForNodeWatcher(ctx context.Context, client client.Client, context *clusterd.Context, opNamespace string) predicate.Funcs {
return predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool {
clientCluster := newClientCluster(client, e.Object.GetNamespace(), context)
return clientCluster.onK8sNode(ctx, e.Object)
return clientCluster.onK8sNode(ctx, e.Object, opNamespace)
},
UpdateFunc: func(e event.UpdateEvent) bool {
@@ -67,7 +67,7 @@ func predicateForNodeWatcher(ctx context.Context, client client.Client, context
}
clientCluster := newClientCluster(client, e.ObjectNew.GetNamespace(), context)
return clientCluster.onK8sNode(ctx, e.ObjectNew)
return clientCluster.onK8sNode(ctx, e.ObjectNew, opNamespace)
}
}
return false
+173 -18
View File
@@ -60,7 +60,8 @@ var (
// drivers that supports fencing, used in naming networkFence object
const (
rbdDriver = "rbd"
rbdDriver = "rbd"
cephfsDriver = "cephfs"
)
func newClientCluster(client client.Client, namespace string, context *clusterd.Context) *clientCluster {
@@ -80,7 +81,7 @@ func checkStorageForNode(cluster *cephv1.CephCluster) bool {
}
// onK8sNode is triggered when a node is added in the Kubernetes cluster
func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bool {
func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, opNamespace string) bool {
node, ok := object.(*corev1.Node)
if !ok {
return false
@@ -90,7 +91,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bo
cluster := c.getCephCluster()
// Continue reconcile in case of failure too since we don't want to block other node reconcile
if err := c.handleNodeFailure(ctx, cluster, node); err != nil {
if err := c.handleNodeFailure(ctx, cluster, node, opNamespace); err != nil {
logger.Errorf("failed to handle node failure. %v", err)
}
@@ -157,7 +158,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bo
return false
}
func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node) error {
func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node, opNamespace string) error {
watchForNodeLoss, err := k8sutil.GetOperatorSetting(ctx, c.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_WATCH_FOR_NODE_FAILURE", "true")
if err != nil {
return pkgerror.Wrapf(err, "failed to get configmap value `ROOK_WATCH_FOR_NODE_FAILURE`.")
@@ -198,7 +199,7 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C
}
if nodeHasOutOfServiceTaint {
err := c.fenceNode(ctx, node, cluster)
err := c.fenceNode(ctx, node, cluster, opNamespace)
if err != nil {
return pkgerror.Wrapf(err, "failed to create network fence for node %q.", node.Name)
}
@@ -210,10 +211,14 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C
return pkgerror.Wrapf(err, "failed to delete rbd network fence for node %q.", node.Name)
}
err = c.unfenceAndDeleteNetworkFence(ctx, *node, cluster, cephfsDriver)
if err != nil {
return pkgerror.Wrapf(err, "failed to delete cephFS network fence for node %q.", node.Name)
}
return nil
}
func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster) error {
func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster, opNamespace string) error {
volumesInuse := node.Status.VolumesInUse
if len(volumesInuse) == 0 {
logger.Debugf("no volumes in use for node %q", node.Name)
@@ -221,9 +226,9 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste
}
logger.Debugf("volumesInuse %s", volumesInuse)
rbdVolumesInUse := getCephVolumesInUse(cluster, volumesInuse)
if len(rbdVolumesInUse) == 0 {
logger.Debugf("no rbd volumes in use for out of service node %q", node.Name)
rbdVolumesInUse, cephFSVolumeInUse := getCephVolumesInUse(cluster, volumesInuse, opNamespace)
if len(rbdVolumesInUse) == 0 && len(cephFSVolumeInUse) == 0 {
logger.Debugf("no rbd or cephfs subvolumes in use for out of service node %q", node.Name)
return nil
}
@@ -233,7 +238,7 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste
}
if len(rbdVolumesInUse) != 0 {
rbdPVList := listRBDPV(listPVs, cluster, rbdVolumesInUse)
rbdPVList := listRBDPV(listPVs, cluster, rbdVolumesInUse, opNamespace)
if len(rbdPVList) == 0 {
logger.Debug("No rbd PVs found on the node")
} else {
@@ -262,22 +267,59 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste
}
}
if len(cephFSVolumeInUse) != 0 {
cephFSVolumeInUseMap := make(map[string]struct{})
for _, vol := range cephFSVolumeInUse {
cephFSVolumeInUseMap[vol] = struct{}{}
}
cephFSPVList := listRWOCephFSPV(listPVs, cluster, cephFSVolumeInUseMap, opNamespace)
if len(cephFSPVList) == 0 {
logger.Debug("No cephfs PVs found on the node %s", node.Name)
return nil
}
logger.Infof("node %q require fencing, found cephfs subvolumes in use", node.Name)
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(c.context, ctx, cluster.Namespace, &cluster.Spec)
if err != nil {
return pkgerror.Wrap(err, "Failed to load cluster info.")
}
for i := range cephFSPVList {
err = c.fenceCephFSSubvolume(ctx, node, cluster, clusterInfo, cephFSPVList[i])
// We only need to create the network fence for any one of cephFS pv.
if err == nil {
break
}
// continue to fence next cephfs subvolume if active client not found
if stderrors.Is(err, errActiveClientNotFound) {
continue
}
if i == len(cephFSPVList)-1 {
return pkgerror.Wrap(err, "failed to fence cephfs subvolumes")
}
logger.Errorf("failed to fence cephfs subvolume %q, trying next cephfs subvolume", cephFSPVList[i].Name)
}
}
return nil
}
func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) []string {
var rbdVolumesInUse []string
func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName, opNamespace string) ([]string, []string) {
var rbdVolumesInUse, cephFSVolumeInUse []string
for _, volume := range volumesInUse {
splitVolumeInUseBased := trimeVolumeInUse(volume)
logger.Infof("volumeInUse after split based on '^' %v", splitVolumeInUseBased)
if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) {
if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", opNamespace) {
rbdVolumesInUse = append(rbdVolumesInUse, splitVolumeInUseBased[1])
}
}
return rbdVolumesInUse
if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.cephfs.csi.ceph.com", opNamespace) {
cephFSVolumeInUse = append(cephFSVolumeInUse, splitVolumeInUseBased[1])
}
}
return rbdVolumesInUse, cephFSVolumeInUse
}
func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string {
@@ -286,7 +328,7 @@ func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string {
return splitVolumeInUseBased
}
func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, rbdVolumesInUse []string) []corev1.PersistentVolume {
func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, rbdVolumesInUse []string, opNamespace string) []corev1.PersistentVolume {
var listRbdPV []corev1.PersistentVolume
for _, pv := range listPVs.Items {
@@ -296,7 +338,7 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster
continue
}
if pv.Spec.CSI.Driver == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) {
if pv.Spec.CSI.Driver == fmt.Sprintf("%s.rbd.csi.ceph.com", opNamespace) {
// Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes.
if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) {
continue
@@ -316,6 +358,35 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster
return listRbdPV
}
func listRWOCephFSPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, cephFSVolumesInUse map[string]struct{}, opNamespace string) []corev1.PersistentVolume {
var listCephFSPV []corev1.PersistentVolume
for _, pv := range listPVs.Items {
// Skip if pv is not provisioned by CSI
if pv.Spec.CSI == nil {
logger.Debugf("PV %q is not provisioned by CSI", pv.Name)
continue
}
if pv.Spec.CSI.Driver == fmt.Sprintf("%s.cephfs.csi.ceph.com", opNamespace) {
// Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes.
if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) {
continue
}
if pv.Spec.CSI.VolumeAttributes["staticVolume"] == "true" {
logger.Debugf("skipping, static PV %q", pv.Name)
continue
}
// Check if the volume is in use
if _, exists := cephFSVolumesInUse[pv.Spec.CSI.VolumeHandle]; exists {
listCephFSPV = append(listCephFSPV, pv)
}
}
}
return listCephFSPV
}
// pvSupportsMultiNodeAccess returns true if the PV access modes contain ReadWriteMany or ReadOnlyMany.
func pvSupportsMultiNodeAccess(accessModes []corev1.PersistentVolumeAccessMode) bool {
for _, accessMode := range accessModes {
@@ -358,6 +429,89 @@ func (c *clientCluster) fenceRbdImage(
return nil
}
func (c *clientCluster) fenceCephFSSubvolume(
ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster,
clusterInfo *cephclient.ClusterInfo, cephFSPV corev1.PersistentVolume) error {
logger.Infof("fencing cephfs subvolume %q on node %q", cephFSPV.Name, node.Name)
status, err := cephclient.StatusWithUser(c.context, clusterInfo)
if err != nil {
return pkgerror.Wrap(err, "failed to get ceph status for check active mds")
}
var activeMDS string
for _, fsRank := range status.Fsmap.ByRank {
if fsRank.Status == "up:active" {
activeMDS = fsRank.Name
}
}
args := []string{"tell", fmt.Sprintf("mds.%s", activeMDS), "client", "ls", "--format", "json"}
cmd := cephclient.NewCephCommand(c.context, clusterInfo, args)
cmd.JsonOutput = true
buf, err := cmd.Run()
if err != nil {
return fmt.Errorf("failed to list watchers for cephfs subvolumeName %s. %v", cephFSPV.Spec.CSI.VolumeAttributes["subvolumeName"], err)
}
ips, err := cephFSMDSClientMarshal(buf, node.Name, cephFSPV)
if err != nil {
return pkgerror.Wrap(err, "failed to unmarshal cephfs mds output")
}
if len(ips) == 0 {
logger.Infof("no active mds clients found for cephfs subvolume %q", cephFSPV.Name)
return errActiveClientNotFound
}
err = c.createNetworkFence(ctx, cephFSPV, node, cluster, ips, cephfsDriver)
if err != nil {
return pkgerror.Wrapf(err, "failed to create network fence for node %q", node.Name)
}
return nil
}
func cephFSMDSClientMarshal(output []byte, nodeName string, cephFSPV corev1.PersistentVolume) ([]string, error) {
type entity struct {
Addr struct {
Addr string `json:"addr"`
Nonce int `json:"nonce"`
} `json:"addr"`
}
type clientMetadata struct {
Root string `json:"root"`
Hostname string `json:"hostname"`
}
type cephFSData struct {
Entity entity `json:"entity"`
ClientMetadata clientMetadata `json:"client_metadata"`
}
var data []cephFSData
err := json.Unmarshal([]byte(output), &data)
if err != nil {
return []string{}, pkgerror.Wrapf(err, "failed to unmarshal cephFS data output")
}
watcherIPlist := []string{}
for _, d := range data {
if d.ClientMetadata.Hostname != "" {
if strings.Contains(nodeName, d.ClientMetadata.Hostname) && cephFSPV.Spec.CSI.VolumeAttributes["subvolumePath"] == d.ClientMetadata.Root {
logger.Infof("cephfs mds client ips to fence %v", d.Entity.Addr)
watcherIP := concatenateWatcherIp(d.Entity.Addr.Addr)
watcherIPlist = append(watcherIPlist, watcherIP)
break
}
}
}
return watcherIPlist, nil
}
func rbdStatusUnMarshal(output []byte) ([]string, error) {
type rbdStatus struct {
Watchers []struct {
@@ -380,7 +534,8 @@ func rbdStatusUnMarshal(output []byte) ([]string, error) {
}
func concatenateWatcherIp(address string) string {
// address is in format `10.63.0.5:0/1254753579` for rbd
// address is in format `10.63.0.5:0/1254753579` for rbd and
// in the format '10.244.0.12:0' for cephfs
// split with separation ':0' to remove nounce and concatenating `/32` to define a network with only one IP address
watcherIP := strings.Split(address, ":0")[0] + "/32"
return watcherIP
+128 -35
View File
@@ -96,6 +96,7 @@ func TestCheckStorageForNode(t *testing.T) {
func TestOnK8sNode(t *testing.T) {
ns := "rook-ceph"
opns := "operator"
ctx := context.TODO()
cephCluster := fakeCluster(ns)
objects := []runtime.Object{
@@ -149,24 +150,25 @@ func TestOnK8sNode(t *testing.T) {
cephCluster.Status.Phase = k8sutil.ReadyStatus
client = getFakeClient(objects...)
clientCluster.client = client
b := clientCluster.onK8sNode(ctx, node)
b := clientCluster.onK8sNode(ctx, node, opns)
assert.True(t, b)
// node will not reconcile
b = clientCluster.onK8sNode(ctx, node)
b = clientCluster.onK8sNode(ctx, node, opns)
assert.False(t, b)
}
func TestHandleNodeFailure(t *testing.T) {
ns := "rook-ceph"
clusterns := "rook-ceph"
opns := "operator"
ctx := context.TODO()
cephCluster := fakeCluster(ns)
cephCluster := fakeCluster(clusterns)
objects := []runtime.Object{
cephCluster,
}
executor := &exectest.MockExecutor{}
client := getFakeClient(objects...)
c := newClientCluster(client, ns, &clusterd.Context{
c := newClientCluster(client, clusterns, &clusterd.Context{
Executor: executor,
Clientset: k8sFake.NewSimpleClientset(),
ApiExtensionsClient: apifake.NewSimpleClientset(),
@@ -179,7 +181,7 @@ func TestHandleNodeFailure(t *testing.T) {
case command == "ceph" && args[0] == "status":
return `{"entity":[{"addr": [{"addr": "10.244.0.12:0", "nonce":3247243972}]}], "client_metadata":{"root":"/"}}`, nil
case command == "ceph" && args[0] == "tell":
return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca"}}]`, nil
return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca","hostname":"fakenode"}}]`, nil
}
return "", errors.Errorf("unexpected rbd/ceph command %q", args)
@@ -205,7 +207,8 @@ func TestHandleNodeFailure(t *testing.T) {
},
},
VolumesInUse: []corev1.UniqueVolumeName{
"kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
"kubernetes.io/csi/operator.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
"kubernetes.io/csi/operator.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
},
},
}
@@ -214,15 +217,15 @@ func TestHandleNodeFailure(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841ca",
Annotations: map[string]string{
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", ns),
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns),
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner",
"volume.kubernetes.io/provisioner-deletion-secret-namespace": ns,
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
},
},
Spec: corev1.PersistentVolumeSpec{
PersistentVolumeSource: corev1.PersistentVolumeSource{
CSI: &corev1.CSIPersistentVolumeSource{
Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", ns),
Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns),
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
VolumeAttributes: map[string]string{
"pool": "replicapool",
@@ -233,19 +236,43 @@ func TestHandleNodeFailure(t *testing.T) {
},
}
staticRbdPV := &corev1.PersistentVolume{
cephfsPV := &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb",
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842ca",
Annotations: map[string]string{
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", ns),
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner",
"volume.kubernetes.io/provisioner-deletion-secret-namespace": ns,
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns),
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner",
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
},
},
Spec: corev1.PersistentVolumeSpec{
PersistentVolumeSource: corev1.PersistentVolumeSource{
CSI: &corev1.CSIPersistentVolumeSource{
Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", ns),
Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns),
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
VolumeAttributes: map[string]string{
"fsName": "myfs",
"subvolumePath": "/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca",
"subvolumeName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca",
},
},
},
},
}
staticRbdPV := &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb",
Annotations: map[string]string{
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns),
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner",
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
},
},
Spec: corev1.PersistentVolumeSpec{
PersistentVolumeSource: corev1.PersistentVolumeSource{
CSI: &corev1.CSIPersistentVolumeSource{
Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns),
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
VolumeAttributes: map[string]string{},
},
@@ -253,19 +280,41 @@ func TestHandleNodeFailure(t *testing.T) {
},
}
staticCephfsPV := &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842cb",
Annotations: map[string]string{
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns),
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner",
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
},
},
Spec: corev1.PersistentVolumeSpec{
PersistentVolumeSource: corev1.PersistentVolumeSource{
CSI: &corev1.CSIPersistentVolumeSource{
Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns),
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
VolumeAttributes: map[string]string{
"staticVolume": "true",
},
},
},
},
}
pvNotProvisionByCSI := &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cc",
Annotations: map[string]string{
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.csi.rbd.com", ns),
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.csi.rbd.com", opns),
"volume.kubernetes.io/provisioner-deletion-secret-name": "csi-rbd-provisioner",
"volume.kubernetes.io/provisioner-deletion-secret-namespace": ns,
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
},
},
Spec: corev1.PersistentVolumeSpec{
PersistentVolumeSource: corev1.PersistentVolumeSource{
CSI: &corev1.CSIPersistentVolumeSource{
Driver: fmt.Sprintf("%s.rbd.csi.com", ns),
Driver: fmt.Sprintf("%s.rbd.csi.com", opns),
VolumeHandle: "0001-0009-csi-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
VolumeAttributes: map[string]string{
"pool": "replicapool",
@@ -285,41 +334,51 @@ func TestHandleNodeFailure(t *testing.T) {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "rook-ceph-mon",
Namespace: ns,
Namespace: clusterns,
},
Data: secrets,
Type: k8sutil.RookType,
}
_, err := c.context.Clientset.CoreV1().Secrets(ns).Create(ctx, secret, metav1.CreateOptions{})
_, err := c.context.Clientset.CoreV1().Secrets(clusterns).Create(ctx, secret, metav1.CreateOptions{})
assert.NoError(t, err)
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, rbdPV, metav1.CreateOptions{})
assert.NoError(t, err)
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, cephfsPV, metav1.CreateOptions{})
assert.NoError(t, err)
_, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &v1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "networkfences.csiaddons.openshift.io"}}, metav1.CreateOptions{})
assert.NoError(t, err)
// When out-of-service taint is added
err = c.handleNodeFailure(ctx, cephCluster, node)
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
assert.NoError(t, err)
networkFenceRbd := &addonsv1alpha1.NetworkFence{}
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns)}, networkFenceRbd)
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns)}, networkFenceRbd)
assert.NoError(t, err)
networkFenceCephFs := &addonsv1alpha1.NetworkFence{}
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns)}, networkFenceCephFs)
assert.NoError(t, err)
networkFences := &addonsv1alpha1.NetworkFenceList{}
err = c.client.List(ctx, networkFences)
assert.NoError(t, err)
var rbdCount int
var rbdCount, cephFsCount int
for _, fence := range networkFences.Items {
// Check if the resource is in the desired namespace
if strings.Contains(fence.Name, rbdDriver) {
rbdCount++
} else if strings.Contains(fence.Name, cephfsDriver) {
cephFsCount++
}
}
assert.Equal(t, 1, rbdCount)
assert.Equal(t, 1, cephFsCount)
// For static rbd pv
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticRbdPV, metav1.CreateOptions{})
@@ -328,14 +387,29 @@ func TestHandleNodeFailure(t *testing.T) {
pvList, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
assert.NoError(t, err)
rbdVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse)
rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse)
rbdVolumesInUse, _ := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns)
rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns)
assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `rbdPV`
err = c.handleNodeFailure(ctx, cephCluster, node)
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
assert.NoError(t, err)
err = c.handleNodeFailure(ctx, cephCluster, node)
// For static cephfs pv
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticCephfsPV, metav1.CreateOptions{})
assert.NoError(t, err)
pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
assert.NoError(t, err)
_, cephFSVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns)
cephFSVolumesInUseMap := make(map[string]struct{})
for _, vol := range cephFSVolumesInUse {
cephFSVolumesInUseMap[vol] = struct{}{}
}
cephFSPVList := listRWOCephFSPV(pvList, cephCluster, cephFSVolumesInUseMap, opns)
assert.Equal(t, len(cephFSPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `cephfsPV`
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
assert.NoError(t, err)
// For pv not provisioned by CSI
@@ -345,25 +419,32 @@ func TestHandleNodeFailure(t *testing.T) {
pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
assert.NoError(t, err)
rbdVolumesInUse = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse)
rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse)
rbdVolumesInUse, _ = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns)
rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns)
assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `PV`
err = c.handleNodeFailure(ctx, cephCluster, node)
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
assert.NoError(t, err)
// When out-of-service taint is removed
node.Spec.Taints = []corev1.Taint{}
networkFenceRbd.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage
err = c.client.Update(ctx, networkFenceRbd)
assert.NoError(t, err)
err = c.handleNodeFailure(ctx, cephCluster, node)
networkFenceCephFs.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage
err = c.client.Update(ctx, networkFenceCephFs)
assert.NoError(t, err)
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns), Namespace: cephCluster.Namespace}, networkFenceRbd)
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
assert.NoError(t, err)
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceRbd)
assert.Error(t, err, kerrors.IsNotFound(err))
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceCephFs)
assert.Error(t, err, kerrors.IsNotFound(err))
}
func TestGetCephVolumesInUse(t *testing.T) {
@@ -371,6 +452,8 @@ func TestGetCephVolumesInUse(t *testing.T) {
volInUse := []corev1.UniqueVolumeName{
"kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
"kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003",
"kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
"kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004",
}
splitVolInUse := trimeVolumeInUse(volInUse[0])
@@ -381,11 +464,21 @@ func TestGetCephVolumesInUse(t *testing.T) {
assert.Equal(t, splitVolInUse[0], "rook-ceph.rbd.csi.ceph.com")
assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003")
trimRbdVolInUse := getCephVolumesInUse(cephCluster, volInUse)
splitVolInUse = trimeVolumeInUse(volInUse[2])
assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com")
assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001")
splitVolInUse = trimeVolumeInUse(volInUse[3])
assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com")
assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004")
trimRbdVolInUse, trimCephFSVolInUse := getCephVolumesInUse(cephCluster, volInUse, cephCluster.Namespace)
expectedRbd := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003"}
expectedCephfs := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004"}
assert.Equal(t, expectedRbd, trimRbdVolInUse)
assert.Equal(t, expectedCephfs, trimCephFSVolInUse)
}
func TestRBDStatusUnMarshal(t *testing.T) {
+3 -3
View File
@@ -102,13 +102,13 @@ var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Conte
// AddToManager adds all the registered controllers to the passed manager.
// each controller package will have an Add method listed in AddToManagerFuncs
// which will setup all the necessary watch
func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context) error {
func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context, opconfig opcontroller.OperatorConfig) error {
if c == nil {
return errors.New("nil context passed")
}
// Run CephCluster CR
if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext); err != nil {
if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext, opconfig); err != nil {
return err
}
@@ -173,7 +173,7 @@ func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan erro
}
// Add the registered controllers to the manager (entrypoint for controllers)
err = o.addToManager(mgr, controllerOpts, context)
err = o.addToManager(mgr, controllerOpts, context, *o.config)
if err != nil {
mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager")
return