diff --git a/pkg/operator/ceph/disruption/nodedrain/add.go b/pkg/operator/ceph/disruption/nodedrain/add.go index 263e99647..c7ee3a121 100644 --- a/pkg/operator/ceph/disruption/nodedrain/add.go +++ b/pkg/operator/ceph/disruption/nodedrain/add.go @@ -20,10 +20,13 @@ import ( "reflect" "time" + "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" + "github.com/rook/rook/pkg/operator/k8sutil" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -67,7 +70,7 @@ func Add(mgr manager.Manager, context *controllerconfig.Context) error { if err != nil { return err } - + // Watch for changes to the node canary deployments err = c.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForOwner{ OwnerType: &corev1.Node{}, IsController: true, @@ -76,5 +79,48 @@ func Add(mgr manager.Manager, context *controllerconfig.Context) error { return err } + // Watch for changes to the osd pod nodename and enqueue thier nodes + err = c.Watch( + &source.Kind{Type: &corev1.Pod{}}, + &handler.EnqueueRequestsFromMapFunc{ + ToRequests: handler.ToRequestsFunc(func(obj handler.MapObject) []reconcile.Request { + pod, ok := obj.Object.(*corev1.Pod) + if !ok { + return []reconcile.Request{} + } + labels := pod.GetLabels() + if appName, ok := labels[k8sutil.AppAttr]; !ok || appName != osd.AppName { + return []reconcile.Request{} + } + nodeName := pod.Spec.NodeName + if len(nodeName) < 1 { + return []reconcile.Request{} + } + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: nodeName}} + return []reconcile.Request{req} + }), + }, + predicate.Funcs{ + UpdateFunc: func(event event.UpdateEvent) bool { + oldPod, ok := event.ObjectOld.(*corev1.Pod) + if !ok { + return false + } + newPod, ok := event.ObjectNew.(*corev1.Pod) + if !ok { + return false + } + // only enqueue if the nodename has changed + if oldPod.Spec.NodeName == newPod.Spec.NodeName { + return false + } + return true + }, + }, + ) + if err != nil { + return err + } + return nil } diff --git a/pkg/operator/ceph/disruption/nodedrain/reconcile.go b/pkg/operator/ceph/disruption/nodedrain/reconcile.go index de0e71764..b09379dab 100644 --- a/pkg/operator/ceph/disruption/nodedrain/reconcile.go +++ b/pkg/operator/ceph/disruption/nodedrain/reconcile.go @@ -22,6 +22,7 @@ import ( "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/disruption/controllerconfig" "github.com/rook/rook/pkg/operator/k8sutil" @@ -82,6 +83,18 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, return reconcile.Result{}, fmt.Errorf("Label key %s does not exist on node %s", corev1.LabelHostname, request.NamespacedName) } + osdPodList := &corev1.PodList{} + err = r.client.List(context.TODO(), osdPodList, client.MatchingLabels{k8sutil.AppAttr: osd.AppName}) + if err != nil { + return reconcile.Result{}, fmt.Errorf("could not list the osd pods: %+v", err) + } + occupiedByOSD := false + for _, osdPod := range osdPodList.Items { + if osdPod.Spec.NodeName == request.Name { + occupiedByOSD = true + break + } + } // Create or Update the deployment default/foo deploy := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -123,11 +136,15 @@ func (r *ReconcileNode) reconcile(request reconcile.Request) (reconcile.Result, return nil } - op, err := controllerutil.CreateOrUpdate(context.TODO(), r.client, deploy, mutateFunc) - if err != nil { - return reconcile.Result{}, fmt.Errorf("Node reconcile failed on op: %s : %+v", op, err) + if occupiedByOSD { + op, err := controllerutil.CreateOrUpdate(context.TODO(), r.client, deploy, mutateFunc) + if err != nil { + return reconcile.Result{}, fmt.Errorf("Node reconcile failed on op: %s : %+v", op, err) + } + logger.Debugf("deployment successfully reconciled for node %s. operation: %s", request.Name, op) + } else { + logger.Debugf("not watching for drains on node %s as there are no osds running there.", request.Name) } - logger.Debugf("Deployment successfully reconciled. operation: %s", op) return reconcile.Result{}, nil }