CSI: set pod anti affinity to provisioner pod

sometimes the kube schedular will schedule the
provisioner pod on the same node. it doesnot make
sense to have both provisioner pod running on the same
node, we need to set the pod anti affinity to make
sure that no provisioner pods runnings on the same
node.

Fixes: #5271

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

pkg/operator/ceph/csi/spec.go

@@ -417,6 +417,8 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers
 		// apply resource request and limit to rbd provisioner containers
 		applyResourcesToContainers(clientset, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec)
 		k8sutil.SetOwnerRef(&rbdProvisionerDeployment.ObjectMeta, ownerRef)
+		antiAffinity := getPodAntiAffinity("app", "csi-rbdplugin-provisioner")
+		rbdProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity
 		err = k8sutil.CreateDeployment("csi-rbdplugin-provisioner", namespace, clientset, rbdProvisionerDeployment)
 		if err != nil {
 			return errors.Wrapf(err, "failed to start rbd provisioner deployment: %+v", rbdProvisionerDeployment)
@@ -461,6 +463,8 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers
 		// apply resource request and limit to cephfs provisioner containers
 		applyResourcesToContainers(clientset, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec)
 		k8sutil.SetOwnerRef(&cephfsProvisionerDeployment.ObjectMeta, ownerRef)
+		antiAffinity := getPodAntiAffinity("app", "csi-cephfsplugin-provisioner")
+		cephfsProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity
 		err = k8sutil.CreateDeployment("csi-cephfsplugin-provisioner", namespace, clientset, cephfsProvisionerDeployment)
 		if err != nil {
 			return errors.Wrapf(err, "failed to start cephfs provisioner deployment: %+v", cephfsProvisionerDeployment)

pkg/operator/ceph/csi/util.go

@@ -30,6 +30,7 @@ import (
 	k8sutil "github.com/rook/rook/pkg/operator/k8sutil"
 	apps "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -219,3 +220,23 @@ func getPortFromConfig(clientset kubernetes.Interface, env string, defaultPort u
 	}
 	return uint16(p), nil
 }
+
+func getPodAntiAffinity(key, value string) corev1.PodAntiAffinity {
+	return corev1.PodAntiAffinity{
+		RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
+			{
+				LabelSelector: &metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      key,
+							Operator: metav1.LabelSelectorOpIn,
+							Values:   []string{value},
+						},
+					},
+				},
+				TopologyKey: corev1.LabelHostname,
+			},
+		},
+	}
+
+}