From 4febbb09b888d79b0f5882fa76cfc5d7fa31e99d Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 13 May 2020 13:52:07 +0530 Subject: [PATCH] CSI: set pod anti affinity to provisioner pod sometimes the kube schedular will schedule the provisioner pod on the same node. it doesnot make sense to have both provisioner pod running on the same node, we need to set the pod anti affinity to make sure that no provisioner pods runnings on the same node. Fixes: #5271 Signed-off-by: Madhu Rajanna --- pkg/operator/ceph/csi/spec.go | 4 ++++ pkg/operator/ceph/csi/util.go | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index a31fb7b64..63e2ffe3c 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -417,6 +417,8 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers // apply resource request and limit to rbd provisioner containers applyResourcesToContainers(clientset, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) k8sutil.SetOwnerRef(&rbdProvisionerDeployment.ObjectMeta, ownerRef) + antiAffinity := getPodAntiAffinity("app", "csi-rbdplugin-provisioner") + rbdProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity err = k8sutil.CreateDeployment("csi-rbdplugin-provisioner", namespace, clientset, rbdProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to start rbd provisioner deployment: %+v", rbdProvisionerDeployment) @@ -461,6 +463,8 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers // apply resource request and limit to cephfs provisioner containers applyResourcesToContainers(clientset, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) k8sutil.SetOwnerRef(&cephfsProvisionerDeployment.ObjectMeta, ownerRef) + antiAffinity := getPodAntiAffinity("app", "csi-cephfsplugin-provisioner") + cephfsProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity err = k8sutil.CreateDeployment("csi-cephfsplugin-provisioner", namespace, clientset, cephfsProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to start cephfs provisioner deployment: %+v", cephfsProvisionerDeployment) diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 866361ce6..48c152250 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -30,6 +30,7 @@ import ( k8sutil "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -219,3 +220,23 @@ func getPortFromConfig(clientset kubernetes.Interface, env string, defaultPort u } return uint16(p), nil } + +func getPodAntiAffinity(key, value string) corev1.PodAntiAffinity { + return corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: key, + Operator: metav1.LabelSelectorOpIn, + Values: []string{value}, + }, + }, + }, + TopologyKey: corev1.LabelHostname, + }, + }, + } + +}