Merge pull request #14258 from subhamkrai/add-cephfs-fencing
mds: block of active mds ip only
This commit is contained in:
5 files changed
+314
-64
No files matched your search
@@ -99,16 +99,17 @@ type ReconcileCephCluster struct {
|
||||
context *clusterd.Context
|
||||
clusterController *ClusterController
|
||||
opManagerContext context.Context
|
||||
opConfig opcontroller.OperatorConfig
|
||||
}
|
||||
|
||||
// Add creates a new CephCluster Controller and adds it to the Manager. The Manager will set fields on the Controller
|
||||
// and Start it when the Manager is Started.
|
||||
func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) error {
|
||||
return add(opManagerContext, mgr, newReconciler(mgr, ctx, clusterController, opManagerContext), ctx)
|
||||
func Add(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
|
||||
return add(opManagerContext, mgr, newReconciler(mgr, ctx, clusterController, opManagerContext, opConfig), ctx, opConfig)
|
||||
}
|
||||
|
||||
// newReconciler returns a new reconcile.Reconciler
|
||||
func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context) reconcile.Reconciler {
|
||||
func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController *ClusterController, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
|
||||
// add "rook-" prefix to the controller name to make sure it is clear to all reading the events
|
||||
// that they are coming from Rook. The controller name already has context that it is for Ceph
|
||||
// and from the cluster controller.
|
||||
@@ -118,12 +119,13 @@ func newReconciler(mgr manager.Manager, ctx *clusterd.Context, clusterController
|
||||
client: mgr.GetClient(),
|
||||
scheme: mgr.GetScheme(),
|
||||
context: ctx,
|
||||
opConfig: opConfig,
|
||||
clusterController: clusterController,
|
||||
opManagerContext: opManagerContext,
|
||||
}
|
||||
}
|
||||
|
||||
func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context) error {
|
||||
func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reconciler, context *clusterd.Context, opConfig opcontroller.OperatorConfig) error {
|
||||
// Create a new controller
|
||||
c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
|
||||
if err != nil {
|
||||
@@ -170,7 +172,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco
|
||||
mgr.GetCache(),
|
||||
&corev1.Node{TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: corev1.SchemeGroupVersion.String()}},
|
||||
handler.EnqueueRequestsFromMapFunc(handlerFunc),
|
||||
predicateForNodeWatcher(opManagerContext, mgr.GetClient(), context))
|
||||
predicateForNodeWatcher(opManagerContext, mgr.GetClient(), context, opConfig.OperatorNamespace))
|
||||
err = c.Watch(nodeKind)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -52,11 +52,11 @@ func shouldReconcileChangedNode(objOld, objNew *corev1.Node) bool {
|
||||
}
|
||||
|
||||
// predicateForNodeWatcher is the predicate function to trigger reconcile on Node events
|
||||
func predicateForNodeWatcher(ctx context.Context, client client.Client, context *clusterd.Context) predicate.Funcs {
|
||||
func predicateForNodeWatcher(ctx context.Context, client client.Client, context *clusterd.Context, opNamespace string) predicate.Funcs {
|
||||
return predicate.Funcs{
|
||||
CreateFunc: func(e event.CreateEvent) bool {
|
||||
clientCluster := newClientCluster(client, e.Object.GetNamespace(), context)
|
||||
return clientCluster.onK8sNode(ctx, e.Object)
|
||||
return clientCluster.onK8sNode(ctx, e.Object, opNamespace)
|
||||
},
|
||||
|
||||
UpdateFunc: func(e event.UpdateEvent) bool {
|
||||
@@ -67,7 +67,7 @@ func predicateForNodeWatcher(ctx context.Context, client client.Client, context
|
||||
}
|
||||
|
||||
clientCluster := newClientCluster(client, e.ObjectNew.GetNamespace(), context)
|
||||
return clientCluster.onK8sNode(ctx, e.ObjectNew)
|
||||
return clientCluster.onK8sNode(ctx, e.ObjectNew, opNamespace)
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -60,7 +60,8 @@ var (
|
||||
|
||||
// drivers that supports fencing, used in naming networkFence object
|
||||
const (
|
||||
rbdDriver = "rbd"
|
||||
rbdDriver = "rbd"
|
||||
cephfsDriver = "cephfs"
|
||||
)
|
||||
|
||||
func newClientCluster(client client.Client, namespace string, context *clusterd.Context) *clientCluster {
|
||||
@@ -80,7 +81,7 @@ func checkStorageForNode(cluster *cephv1.CephCluster) bool {
|
||||
}
|
||||
|
||||
// onK8sNode is triggered when a node is added in the Kubernetes cluster
|
||||
func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bool {
|
||||
func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, opNamespace string) bool {
|
||||
node, ok := object.(*corev1.Node)
|
||||
if !ok {
|
||||
return false
|
||||
@@ -90,7 +91,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bo
|
||||
cluster := c.getCephCluster()
|
||||
|
||||
// Continue reconcile in case of failure too since we don't want to block other node reconcile
|
||||
if err := c.handleNodeFailure(ctx, cluster, node); err != nil {
|
||||
if err := c.handleNodeFailure(ctx, cluster, node, opNamespace); err != nil {
|
||||
logger.Errorf("failed to handle node failure. %v", err)
|
||||
}
|
||||
|
||||
@@ -157,7 +158,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object) bo
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node) error {
|
||||
func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node, opNamespace string) error {
|
||||
watchForNodeLoss, err := k8sutil.GetOperatorSetting(ctx, c.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_WATCH_FOR_NODE_FAILURE", "true")
|
||||
if err != nil {
|
||||
return pkgerror.Wrapf(err, "failed to get configmap value `ROOK_WATCH_FOR_NODE_FAILURE`.")
|
||||
@@ -198,7 +199,7 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C
|
||||
}
|
||||
|
||||
if nodeHasOutOfServiceTaint {
|
||||
err := c.fenceNode(ctx, node, cluster)
|
||||
err := c.fenceNode(ctx, node, cluster, opNamespace)
|
||||
if err != nil {
|
||||
return pkgerror.Wrapf(err, "failed to create network fence for node %q.", node.Name)
|
||||
}
|
||||
@@ -210,10 +211,14 @@ func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.C
|
||||
return pkgerror.Wrapf(err, "failed to delete rbd network fence for node %q.", node.Name)
|
||||
}
|
||||
|
||||
err = c.unfenceAndDeleteNetworkFence(ctx, *node, cluster, cephfsDriver)
|
||||
if err != nil {
|
||||
return pkgerror.Wrapf(err, "failed to delete cephFS network fence for node %q.", node.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster) error {
|
||||
func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster, opNamespace string) error {
|
||||
volumesInuse := node.Status.VolumesInUse
|
||||
if len(volumesInuse) == 0 {
|
||||
logger.Debugf("no volumes in use for node %q", node.Name)
|
||||
@@ -221,9 +226,9 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste
|
||||
}
|
||||
logger.Debugf("volumesInuse %s", volumesInuse)
|
||||
|
||||
rbdVolumesInUse := getCephVolumesInUse(cluster, volumesInuse)
|
||||
if len(rbdVolumesInUse) == 0 {
|
||||
logger.Debugf("no rbd volumes in use for out of service node %q", node.Name)
|
||||
rbdVolumesInUse, cephFSVolumeInUse := getCephVolumesInUse(cluster, volumesInuse, opNamespace)
|
||||
if len(rbdVolumesInUse) == 0 && len(cephFSVolumeInUse) == 0 {
|
||||
logger.Debugf("no rbd or cephfs subvolumes in use for out of service node %q", node.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -233,7 +238,7 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste
|
||||
}
|
||||
|
||||
if len(rbdVolumesInUse) != 0 {
|
||||
rbdPVList := listRBDPV(listPVs, cluster, rbdVolumesInUse)
|
||||
rbdPVList := listRBDPV(listPVs, cluster, rbdVolumesInUse, opNamespace)
|
||||
if len(rbdPVList) == 0 {
|
||||
logger.Debug("No rbd PVs found on the node")
|
||||
} else {
|
||||
@@ -262,22 +267,59 @@ func (c *clientCluster) fenceNode(ctx context.Context, node *corev1.Node, cluste
|
||||
}
|
||||
}
|
||||
|
||||
if len(cephFSVolumeInUse) != 0 {
|
||||
cephFSVolumeInUseMap := make(map[string]struct{})
|
||||
for _, vol := range cephFSVolumeInUse {
|
||||
cephFSVolumeInUseMap[vol] = struct{}{}
|
||||
}
|
||||
cephFSPVList := listRWOCephFSPV(listPVs, cluster, cephFSVolumeInUseMap, opNamespace)
|
||||
if len(cephFSPVList) == 0 {
|
||||
logger.Debug("No cephfs PVs found on the node %s", node.Name)
|
||||
return nil
|
||||
}
|
||||
logger.Infof("node %q require fencing, found cephfs subvolumes in use", node.Name)
|
||||
clusterInfo, _, _, err := opcontroller.LoadClusterInfo(c.context, ctx, cluster.Namespace, &cluster.Spec)
|
||||
if err != nil {
|
||||
return pkgerror.Wrap(err, "Failed to load cluster info.")
|
||||
}
|
||||
|
||||
for i := range cephFSPVList {
|
||||
err = c.fenceCephFSSubvolume(ctx, node, cluster, clusterInfo, cephFSPVList[i])
|
||||
// We only need to create the network fence for any one of cephFS pv.
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
// continue to fence next cephfs subvolume if active client not found
|
||||
if stderrors.Is(err, errActiveClientNotFound) {
|
||||
continue
|
||||
}
|
||||
if i == len(cephFSPVList)-1 {
|
||||
return pkgerror.Wrap(err, "failed to fence cephfs subvolumes")
|
||||
}
|
||||
logger.Errorf("failed to fence cephfs subvolume %q, trying next cephfs subvolume", cephFSPVList[i].Name)
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName) []string {
|
||||
var rbdVolumesInUse []string
|
||||
func getCephVolumesInUse(cluster *cephv1.CephCluster, volumesInUse []corev1.UniqueVolumeName, opNamespace string) ([]string, []string) {
|
||||
var rbdVolumesInUse, cephFSVolumeInUse []string
|
||||
|
||||
for _, volume := range volumesInUse {
|
||||
splitVolumeInUseBased := trimeVolumeInUse(volume)
|
||||
logger.Infof("volumeInUse after split based on '^' %v", splitVolumeInUseBased)
|
||||
|
||||
if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) {
|
||||
if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.rbd.csi.ceph.com", opNamespace) {
|
||||
rbdVolumesInUse = append(rbdVolumesInUse, splitVolumeInUseBased[1])
|
||||
}
|
||||
}
|
||||
|
||||
return rbdVolumesInUse
|
||||
if len(splitVolumeInUseBased) == 2 && splitVolumeInUseBased[0] == fmt.Sprintf("%s.cephfs.csi.ceph.com", opNamespace) {
|
||||
cephFSVolumeInUse = append(cephFSVolumeInUse, splitVolumeInUseBased[1])
|
||||
}
|
||||
}
|
||||
return rbdVolumesInUse, cephFSVolumeInUse
|
||||
}
|
||||
|
||||
func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string {
|
||||
@@ -286,7 +328,7 @@ func trimeVolumeInUse(volume corev1.UniqueVolumeName) []string {
|
||||
return splitVolumeInUseBased
|
||||
}
|
||||
|
||||
func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, rbdVolumesInUse []string) []corev1.PersistentVolume {
|
||||
func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, rbdVolumesInUse []string, opNamespace string) []corev1.PersistentVolume {
|
||||
var listRbdPV []corev1.PersistentVolume
|
||||
|
||||
for _, pv := range listPVs.Items {
|
||||
@@ -296,7 +338,7 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster
|
||||
continue
|
||||
}
|
||||
|
||||
if pv.Spec.CSI.Driver == fmt.Sprintf("%s.rbd.csi.ceph.com", cluster.Namespace) {
|
||||
if pv.Spec.CSI.Driver == fmt.Sprintf("%s.rbd.csi.ceph.com", opNamespace) {
|
||||
// Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes.
|
||||
if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) {
|
||||
continue
|
||||
@@ -316,6 +358,35 @@ func listRBDPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster
|
||||
return listRbdPV
|
||||
}
|
||||
|
||||
func listRWOCephFSPV(listPVs *corev1.PersistentVolumeList, cluster *cephv1.CephCluster, cephFSVolumesInUse map[string]struct{}, opNamespace string) []corev1.PersistentVolume {
|
||||
var listCephFSPV []corev1.PersistentVolume
|
||||
|
||||
for _, pv := range listPVs.Items {
|
||||
// Skip if pv is not provisioned by CSI
|
||||
if pv.Spec.CSI == nil {
|
||||
logger.Debugf("PV %q is not provisioned by CSI", pv.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
if pv.Spec.CSI.Driver == fmt.Sprintf("%s.cephfs.csi.ceph.com", opNamespace) {
|
||||
// Ignore PVs that support multinode access (RWX, ROX), since they can be mounted on multiple nodes.
|
||||
if pvSupportsMultiNodeAccess(pv.Spec.AccessModes) {
|
||||
continue
|
||||
}
|
||||
|
||||
if pv.Spec.CSI.VolumeAttributes["staticVolume"] == "true" {
|
||||
logger.Debugf("skipping, static PV %q", pv.Name)
|
||||
continue
|
||||
}
|
||||
// Check if the volume is in use
|
||||
if _, exists := cephFSVolumesInUse[pv.Spec.CSI.VolumeHandle]; exists {
|
||||
listCephFSPV = append(listCephFSPV, pv)
|
||||
}
|
||||
}
|
||||
}
|
||||
return listCephFSPV
|
||||
}
|
||||
|
||||
// pvSupportsMultiNodeAccess returns true if the PV access modes contain ReadWriteMany or ReadOnlyMany.
|
||||
func pvSupportsMultiNodeAccess(accessModes []corev1.PersistentVolumeAccessMode) bool {
|
||||
for _, accessMode := range accessModes {
|
||||
@@ -358,6 +429,89 @@ func (c *clientCluster) fenceRbdImage(
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *clientCluster) fenceCephFSSubvolume(
|
||||
ctx context.Context, node *corev1.Node, cluster *cephv1.CephCluster,
|
||||
clusterInfo *cephclient.ClusterInfo, cephFSPV corev1.PersistentVolume) error {
|
||||
|
||||
logger.Infof("fencing cephfs subvolume %q on node %q", cephFSPV.Name, node.Name)
|
||||
|
||||
status, err := cephclient.StatusWithUser(c.context, clusterInfo)
|
||||
if err != nil {
|
||||
return pkgerror.Wrap(err, "failed to get ceph status for check active mds")
|
||||
}
|
||||
|
||||
var activeMDS string
|
||||
for _, fsRank := range status.Fsmap.ByRank {
|
||||
if fsRank.Status == "up:active" {
|
||||
activeMDS = fsRank.Name
|
||||
}
|
||||
}
|
||||
|
||||
args := []string{"tell", fmt.Sprintf("mds.%s", activeMDS), "client", "ls", "--format", "json"}
|
||||
cmd := cephclient.NewCephCommand(c.context, clusterInfo, args)
|
||||
cmd.JsonOutput = true
|
||||
|
||||
buf, err := cmd.Run()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list watchers for cephfs subvolumeName %s. %v", cephFSPV.Spec.CSI.VolumeAttributes["subvolumeName"], err)
|
||||
}
|
||||
ips, err := cephFSMDSClientMarshal(buf, node.Name, cephFSPV)
|
||||
if err != nil {
|
||||
return pkgerror.Wrap(err, "failed to unmarshal cephfs mds output")
|
||||
}
|
||||
|
||||
if len(ips) == 0 {
|
||||
logger.Infof("no active mds clients found for cephfs subvolume %q", cephFSPV.Name)
|
||||
return errActiveClientNotFound
|
||||
}
|
||||
|
||||
err = c.createNetworkFence(ctx, cephFSPV, node, cluster, ips, cephfsDriver)
|
||||
if err != nil {
|
||||
return pkgerror.Wrapf(err, "failed to create network fence for node %q", node.Name)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cephFSMDSClientMarshal(output []byte, nodeName string, cephFSPV corev1.PersistentVolume) ([]string, error) {
|
||||
type entity struct {
|
||||
Addr struct {
|
||||
Addr string `json:"addr"`
|
||||
Nonce int `json:"nonce"`
|
||||
} `json:"addr"`
|
||||
}
|
||||
|
||||
type clientMetadata struct {
|
||||
Root string `json:"root"`
|
||||
Hostname string `json:"hostname"`
|
||||
}
|
||||
|
||||
type cephFSData struct {
|
||||
Entity entity `json:"entity"`
|
||||
ClientMetadata clientMetadata `json:"client_metadata"`
|
||||
}
|
||||
|
||||
var data []cephFSData
|
||||
err := json.Unmarshal([]byte(output), &data)
|
||||
if err != nil {
|
||||
return []string{}, pkgerror.Wrapf(err, "failed to unmarshal cephFS data output")
|
||||
}
|
||||
|
||||
watcherIPlist := []string{}
|
||||
for _, d := range data {
|
||||
if d.ClientMetadata.Hostname != "" {
|
||||
if strings.Contains(nodeName, d.ClientMetadata.Hostname) && cephFSPV.Spec.CSI.VolumeAttributes["subvolumePath"] == d.ClientMetadata.Root {
|
||||
logger.Infof("cephfs mds client ips to fence %v", d.Entity.Addr)
|
||||
watcherIP := concatenateWatcherIp(d.Entity.Addr.Addr)
|
||||
watcherIPlist = append(watcherIPlist, watcherIP)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return watcherIPlist, nil
|
||||
}
|
||||
|
||||
func rbdStatusUnMarshal(output []byte) ([]string, error) {
|
||||
type rbdStatus struct {
|
||||
Watchers []struct {
|
||||
@@ -380,7 +534,8 @@ func rbdStatusUnMarshal(output []byte) ([]string, error) {
|
||||
}
|
||||
|
||||
func concatenateWatcherIp(address string) string {
|
||||
// address is in format `10.63.0.5:0/1254753579` for rbd
|
||||
// address is in format `10.63.0.5:0/1254753579` for rbd and
|
||||
// in the format '10.244.0.12:0' for cephfs
|
||||
// split with separation ':0' to remove nounce and concatenating `/32` to define a network with only one IP address
|
||||
watcherIP := strings.Split(address, ":0")[0] + "/32"
|
||||
return watcherIP
|
||||
|
||||
@@ -96,6 +96,7 @@ func TestCheckStorageForNode(t *testing.T) {
|
||||
|
||||
func TestOnK8sNode(t *testing.T) {
|
||||
ns := "rook-ceph"
|
||||
opns := "operator"
|
||||
ctx := context.TODO()
|
||||
cephCluster := fakeCluster(ns)
|
||||
objects := []runtime.Object{
|
||||
@@ -149,24 +150,25 @@ func TestOnK8sNode(t *testing.T) {
|
||||
cephCluster.Status.Phase = k8sutil.ReadyStatus
|
||||
client = getFakeClient(objects...)
|
||||
clientCluster.client = client
|
||||
b := clientCluster.onK8sNode(ctx, node)
|
||||
b := clientCluster.onK8sNode(ctx, node, opns)
|
||||
assert.True(t, b)
|
||||
|
||||
// node will not reconcile
|
||||
b = clientCluster.onK8sNode(ctx, node)
|
||||
b = clientCluster.onK8sNode(ctx, node, opns)
|
||||
assert.False(t, b)
|
||||
}
|
||||
|
||||
func TestHandleNodeFailure(t *testing.T) {
|
||||
ns := "rook-ceph"
|
||||
clusterns := "rook-ceph"
|
||||
opns := "operator"
|
||||
ctx := context.TODO()
|
||||
cephCluster := fakeCluster(ns)
|
||||
cephCluster := fakeCluster(clusterns)
|
||||
objects := []runtime.Object{
|
||||
cephCluster,
|
||||
}
|
||||
executor := &exectest.MockExecutor{}
|
||||
client := getFakeClient(objects...)
|
||||
c := newClientCluster(client, ns, &clusterd.Context{
|
||||
c := newClientCluster(client, clusterns, &clusterd.Context{
|
||||
Executor: executor,
|
||||
Clientset: k8sFake.NewSimpleClientset(),
|
||||
ApiExtensionsClient: apifake.NewSimpleClientset(),
|
||||
@@ -179,7 +181,7 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
case command == "ceph" && args[0] == "status":
|
||||
return `{"entity":[{"addr": [{"addr": "10.244.0.12:0", "nonce":3247243972}]}], "client_metadata":{"root":"/"}}`, nil
|
||||
case command == "ceph" && args[0] == "tell":
|
||||
return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca"}}]`, nil
|
||||
return `[{"entity":{"addr":{"addr":"10.244.0.12:0","nonce":3247243972}}, "client_metadata":{"root":"/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca","hostname":"fakenode"}}]`, nil
|
||||
|
||||
}
|
||||
return "", errors.Errorf("unexpected rbd/ceph command %q", args)
|
||||
@@ -205,7 +207,8 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
},
|
||||
},
|
||||
VolumesInUse: []corev1.UniqueVolumeName{
|
||||
"kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
|
||||
"kubernetes.io/csi/operator.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
|
||||
"kubernetes.io/csi/operator.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -214,15 +217,15 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841ca",
|
||||
Annotations: map[string]string{
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", ns),
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns),
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner",
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": ns,
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PersistentVolumeSpec{
|
||||
PersistentVolumeSource: corev1.PersistentVolumeSource{
|
||||
CSI: &corev1.CSIPersistentVolumeSource{
|
||||
Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", ns),
|
||||
Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns),
|
||||
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
|
||||
VolumeAttributes: map[string]string{
|
||||
"pool": "replicapool",
|
||||
@@ -233,19 +236,43 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
staticRbdPV := &corev1.PersistentVolume{
|
||||
cephfsPV := &corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb",
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842ca",
|
||||
Annotations: map[string]string{
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", ns),
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner",
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": ns,
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns),
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner",
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PersistentVolumeSpec{
|
||||
PersistentVolumeSource: corev1.PersistentVolumeSource{
|
||||
CSI: &corev1.CSIPersistentVolumeSource{
|
||||
Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", ns),
|
||||
Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns),
|
||||
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
|
||||
VolumeAttributes: map[string]string{
|
||||
"fsName": "myfs",
|
||||
"subvolumePath": "/volumes/csi/csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca",
|
||||
"subvolumeName": "csi-vol-58469d41-f6c0-4720-b23a-0a0826b842ca",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
staticRbdPV := &corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cb",
|
||||
Annotations: map[string]string{
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.rbd.csi.ceph.com", opns),
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-rbd-provisioner",
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PersistentVolumeSpec{
|
||||
PersistentVolumeSource: corev1.PersistentVolumeSource{
|
||||
CSI: &corev1.CSIPersistentVolumeSource{
|
||||
Driver: fmt.Sprintf("%s.rbd.csi.ceph.com", opns),
|
||||
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
|
||||
VolumeAttributes: map[string]string{},
|
||||
},
|
||||
@@ -253,19 +280,41 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
staticCephfsPV := &corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b842cb",
|
||||
Annotations: map[string]string{
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.cephfs.csi.ceph.com", opns),
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-name": "rook-csi-cephfs-provisioner",
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PersistentVolumeSpec{
|
||||
PersistentVolumeSource: corev1.PersistentVolumeSource{
|
||||
CSI: &corev1.CSIPersistentVolumeSource{
|
||||
Driver: fmt.Sprintf("%s.cephfs.csi.ceph.com", opns),
|
||||
VolumeHandle: "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
|
||||
VolumeAttributes: map[string]string{
|
||||
"staticVolume": "true",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pvNotProvisionByCSI := &corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pvc-58469d41-f6c0-4720-b23a-0a0826b841cc",
|
||||
Annotations: map[string]string{
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.csi.rbd.com", ns),
|
||||
"pv.kubernetes.io/provisioned-by": fmt.Sprintf("%s.csi.rbd.com", opns),
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-name": "csi-rbd-provisioner",
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": ns,
|
||||
"volume.kubernetes.io/provisioner-deletion-secret-namespace": clusterns,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PersistentVolumeSpec{
|
||||
PersistentVolumeSource: corev1.PersistentVolumeSource{
|
||||
CSI: &corev1.CSIPersistentVolumeSource{
|
||||
Driver: fmt.Sprintf("%s.rbd.csi.com", ns),
|
||||
Driver: fmt.Sprintf("%s.rbd.csi.com", opns),
|
||||
VolumeHandle: "0001-0009-csi-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
|
||||
VolumeAttributes: map[string]string{
|
||||
"pool": "replicapool",
|
||||
@@ -285,41 +334,51 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
secret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-ceph-mon",
|
||||
Namespace: ns,
|
||||
Namespace: clusterns,
|
||||
},
|
||||
Data: secrets,
|
||||
Type: k8sutil.RookType,
|
||||
}
|
||||
_, err := c.context.Clientset.CoreV1().Secrets(ns).Create(ctx, secret, metav1.CreateOptions{})
|
||||
_, err := c.context.Clientset.CoreV1().Secrets(clusterns).Create(ctx, secret, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, rbdPV, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, cephfsPV, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &v1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "networkfences.csiaddons.openshift.io"}}, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// When out-of-service taint is added
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node)
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
|
||||
assert.NoError(t, err)
|
||||
|
||||
networkFenceRbd := &addonsv1alpha1.NetworkFence{}
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns)}, networkFenceRbd)
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns)}, networkFenceRbd)
|
||||
assert.NoError(t, err)
|
||||
|
||||
networkFenceCephFs := &addonsv1alpha1.NetworkFence{}
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns)}, networkFenceCephFs)
|
||||
assert.NoError(t, err)
|
||||
|
||||
networkFences := &addonsv1alpha1.NetworkFenceList{}
|
||||
err = c.client.List(ctx, networkFences)
|
||||
assert.NoError(t, err)
|
||||
var rbdCount int
|
||||
var rbdCount, cephFsCount int
|
||||
|
||||
for _, fence := range networkFences.Items {
|
||||
// Check if the resource is in the desired namespace
|
||||
if strings.Contains(fence.Name, rbdDriver) {
|
||||
rbdCount++
|
||||
} else if strings.Contains(fence.Name, cephfsDriver) {
|
||||
cephFsCount++
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, rbdCount)
|
||||
assert.Equal(t, 1, cephFsCount)
|
||||
|
||||
// For static rbd pv
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticRbdPV, metav1.CreateOptions{})
|
||||
@@ -328,14 +387,29 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
pvList, err := c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
rbdVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse)
|
||||
rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse)
|
||||
rbdVolumesInUse, _ := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns)
|
||||
rbdPVList := listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns)
|
||||
assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `rbdPV`
|
||||
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node)
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node)
|
||||
// For static cephfs pv
|
||||
_, err = c.context.Clientset.CoreV1().PersistentVolumes().Create(ctx, staticCephfsPV, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, cephFSVolumesInUse := getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns)
|
||||
cephFSVolumesInUseMap := make(map[string]struct{})
|
||||
for _, vol := range cephFSVolumesInUse {
|
||||
cephFSVolumesInUseMap[vol] = struct{}{}
|
||||
}
|
||||
cephFSPVList := listRWOCephFSPV(pvList, cephCluster, cephFSVolumesInUseMap, opns)
|
||||
assert.Equal(t, len(cephFSPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `cephfsPV`
|
||||
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// For pv not provisioned by CSI
|
||||
@@ -345,25 +419,32 @@ func TestHandleNodeFailure(t *testing.T) {
|
||||
pvList, err = c.context.Clientset.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
rbdVolumesInUse = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse)
|
||||
rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse)
|
||||
rbdVolumesInUse, _ = getCephVolumesInUse(cephCluster, node.Status.VolumesInUse, opns)
|
||||
rbdPVList = listRBDPV(pvList, cephCluster, rbdVolumesInUse, opns)
|
||||
assert.Equal(t, len(rbdPVList), 1) // it will be equal to one since we have one pv provisioned by csi named `PV`
|
||||
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node)
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// When out-of-service taint is removed
|
||||
node.Spec.Taints = []corev1.Taint{}
|
||||
|
||||
networkFenceRbd.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage
|
||||
err = c.client.Update(ctx, networkFenceRbd)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node)
|
||||
networkFenceCephFs.Status.Message = addonsv1alpha1.UnFenceOperationSuccessfulMessage
|
||||
err = c.client.Update(ctx, networkFenceCephFs)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, ns), Namespace: cephCluster.Namespace}, networkFenceRbd)
|
||||
err = c.handleNodeFailure(ctx, cephCluster, node, opns)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, rbdDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceRbd)
|
||||
assert.Error(t, err, kerrors.IsNotFound(err))
|
||||
|
||||
err = c.client.Get(ctx, types.NamespacedName{Name: fenceResourceName(node.Name, cephfsDriver, clusterns), Namespace: cephCluster.Namespace}, networkFenceCephFs)
|
||||
assert.Error(t, err, kerrors.IsNotFound(err))
|
||||
|
||||
}
|
||||
|
||||
func TestGetCephVolumesInUse(t *testing.T) {
|
||||
@@ -371,6 +452,8 @@ func TestGetCephVolumesInUse(t *testing.T) {
|
||||
volInUse := []corev1.UniqueVolumeName{
|
||||
"kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002",
|
||||
"kubernetes.io/csi/rook-ceph.rbd.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003",
|
||||
"kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001",
|
||||
"kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com^0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004",
|
||||
}
|
||||
|
||||
splitVolInUse := trimeVolumeInUse(volInUse[0])
|
||||
@@ -381,11 +464,21 @@ func TestGetCephVolumesInUse(t *testing.T) {
|
||||
assert.Equal(t, splitVolInUse[0], "rook-ceph.rbd.csi.ceph.com")
|
||||
assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003")
|
||||
|
||||
trimRbdVolInUse := getCephVolumesInUse(cephCluster, volInUse)
|
||||
splitVolInUse = trimeVolumeInUse(volInUse[2])
|
||||
assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com")
|
||||
assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001")
|
||||
|
||||
splitVolInUse = trimeVolumeInUse(volInUse[3])
|
||||
assert.Equal(t, splitVolInUse[0], "rook-ceph.cephfs.csi.ceph.com")
|
||||
assert.Equal(t, splitVolInUse[1], "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004")
|
||||
|
||||
trimRbdVolInUse, trimCephFSVolInUse := getCephVolumesInUse(cephCluster, volInUse, cephCluster.Namespace)
|
||||
|
||||
expectedRbd := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4002", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4003"}
|
||||
expectedCephfs := []string{"0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4001", "0001-0009-rook-ceph-0000000000000002-24862838-240d-4215-9183-abfc0e9e4004"}
|
||||
|
||||
assert.Equal(t, expectedRbd, trimRbdVolInUse)
|
||||
assert.Equal(t, expectedCephfs, trimCephFSVolInUse)
|
||||
}
|
||||
|
||||
func TestRBDStatusUnMarshal(t *testing.T) {
|
||||
|
||||
@@ -102,13 +102,13 @@ var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Conte
|
||||
// AddToManager adds all the registered controllers to the passed manager.
|
||||
// each controller package will have an Add method listed in AddToManagerFuncs
|
||||
// which will setup all the necessary watch
|
||||
func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context) error {
|
||||
func (o *Operator) addToManager(m manager.Manager, c *controllerconfig.Context, opManagerContext context.Context, opconfig opcontroller.OperatorConfig) error {
|
||||
if c == nil {
|
||||
return errors.New("nil context passed")
|
||||
}
|
||||
|
||||
// Run CephCluster CR
|
||||
if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext); err != nil {
|
||||
if err := cluster.Add(m, c.ClusterdContext, o.clusterController, opManagerContext, opconfig); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -173,7 +173,7 @@ func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan erro
|
||||
}
|
||||
|
||||
// Add the registered controllers to the manager (entrypoint for controllers)
|
||||
err = o.addToManager(mgr, controllerOpts, context)
|
||||
err = o.addToManager(mgr, controllerOpts, context, *o.config)
|
||||
if err != nil {
|
||||
mgrErrorCh <- errors.Wrap(err, "failed to add controllers to controller-runtime manager")
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user