Use Deployment with leader election instead of StatefulSet

Deployment behaves better when a node gets disconnected
from the rest of the cluster - new provisioner leader
is elected in ~15 seconds, while it may take up to
5 minutes for StatefulSet to start a new replica.

if kube version is 1.13.x deploy provisioner as statefulset.
if kube version is higher than 1.14+ deploy provisioner
as deployment.

Refer: kubernetes-csi/external-provisioner@52d1fbc
Refer: ceph/ceph-csi#497

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

Gopkg.lock

@@ -1465,6 +1465,7 @@
     "k8s.io/apimachinery/pkg/util/version",
     "k8s.io/apimachinery/pkg/util/wait",
     "k8s.io/apimachinery/pkg/util/yaml",
+    "k8s.io/apimachinery/pkg/version",
     "k8s.io/apimachinery/pkg/watch",
     "k8s.io/apiserver/pkg/server",
     "k8s.io/client-go/discovery",

cluster/examples/kubernetes/ceph/common.yaml

@@ -531,6 +531,7 @@ rules:
   resources:
   - daemonsets
   - statefulsets
+  - deployments
   verbs:
   - get
   - list
@@ -1156,6 +1157,9 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 # OLM: END CSI CEPHFS ROLE
 # OLM: BEGIN CSI CEPHFS ROLEBINDING
 ---
@@ -1330,6 +1334,9 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 # OLM: END CSI RBD ROLE
 # OLM: BEGIN CSI RBD ROLEBINDING
 ---
@@ -1434,7 +1441,10 @@ rules:
     verbs: ["get", "list", "watch"]
   - apiGroups: ["apiextensions.k8s.io"]
     resources: ["customresourcedefinitions"]
-    verbs: ["create"]
+    verbs: ["create", "list", "watch", "delete", "get", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
 # OLM: END CSI RBD CLUSTER ROLE
 # OLM: BEGIN CSI RBD CLUSTER ROLEBINDING
 ---

cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml

@@ -0,0 +1,149 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-cephfsplugin-provisioner
+  namespace: {{ .Namespace }}
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+     app: csi-cephfsplugin-provisioner
+  template:
+    metadata:
+      labels:
+        app: csi-cephfsplugin-provisioner
+    spec:
+      serviceAccount: rook-csi-cephfs-provisioner-sa
+      containers:
+        - name: csi-attacher
+          image: {{ .AttacherImage }}
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election=true"
+            - "--leader-election-type=leases"
+            - "--leader-election-namespace={{ .Namespace }}"
+          env:
+            - name: ADDRESS
+              value: /csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-provisioner
+          image: {{ .ProvisionerImage }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=60s"
+            - "--retry-interval-start=500ms"
+            - "--enable-leader-election=true"
+            - "--leader-election-type=leases"
+            - "--leader-election-namespace={{ .Namespace }}"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-cephfsplugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+          image: {{ .CSIPluginImage }}
+          args:
+            - "--nodeid=$(NODE_ID)"
+            - "--type=cephfs"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--v=5"
+            - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
+            - "--metadatastorage=k8s_configmap"
+            - "--pidlimit=-1"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: host-sys
+              mountPath: /sys
+            - name: lib-modules
+              mountPath: /lib/modules
+              readOnly: true
+            - name: host-dev
+              mountPath: /dev
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+        - name: liveness-prometheus
+          image: {{ .CSIPluginImage }}
+          args:
+            - "--type=liveness"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--livenessport=8081"
+            - "--livenesspath=/metrics"
+            - "--polltime=60s"
+            - "--timeout=3s"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          imagePullPolicy: "IfNotPresent"
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com
+            type: DirectoryOrCreate
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: ceph-csi-config
+          configMap:
+            name: rook-ceph-csi-config
+            items:
+              - key: csi-cluster-config-json
+                path: config.json
+        - name: keys-tmp-dir
+          emptyDir: {
+            medium: "Memory"
+          }
+---
+# This is a service to expose the liveness side car
+apiVersion: v1
+kind: Service
+metadata:
+  name: csi-liveness-cephfsplugin-provisioner
+  labels:
+    app: csi-liveness
+spec:
+  ports:
+    - name: http-metrics
+      port: 8080
+      protocol: TCP
+      targetPort: 8081
+  selector:
+    app: csi-cephfsplugin-provisioner

cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml

@@ -0,0 +1,173 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-rbdplugin-provisioner
+  namespace: {{ .Namespace }}
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+     app: csi-rbdplugin-provisioner
+  template:
+    metadata:
+      labels:
+        app: csi-rbdplugin-provisioner
+    spec:
+      serviceAccount: rook-csi-rbd-provisioner-sa
+      containers:
+        - name: csi-provisioner
+          image: {{ .ProvisionerImage }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=60s"
+            - "--retry-interval-start=500ms"
+            - "--enable-leader-election=true"
+            - "--leader-election-type=leases"
+            - "--leader-election-namespace={{ .Namespace }}"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-rbdplugin-attacher
+          image: {{ .AttacherImage }}
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election=true"
+            - "--leader-election-type=leases"
+            - "--leader-election-namespace={{ .Namespace }}"
+          env:
+            - name: ADDRESS
+              value: /csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-snapshotter
+          image:  {{ .SnapshotterImage }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=60s"
+            - "--leader-election=true"
+            - "--leader-election-namespace={{ .Namespace }}"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-rbdplugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+          image: {{ .CSIPluginImage }}
+          args :
+            - "--nodeid=$(NODE_ID)"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--v=5"
+            - "--type=rbd"
+            - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
+            - "--containerized=true"
+            - "--pidlimit=-1"
+          env:
+            - name: HOST_ROOTFS
+              value: "/rootfs"
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - mountPath: /dev
+              name: host-dev
+            - mountPath: /rootfs
+              name: host-rootfs
+            - mountPath: /sys
+              name: host-sys
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+        - name: liveness-prometheus
+          image: {{ .CSIPluginImage }}
+          args:
+            - "--type=liveness"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--livenessport=8080"
+            - "--livenesspath=/metrics"
+            - "--polltime=60s"
+            - "--timeout=3s"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          imagePullPolicy: "IfNotPresent"
+      volumes:
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: host-rootfs
+          hostPath:
+            path: /
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com
+            type: DirectoryOrCreate
+        - name: ceph-csi-config
+          configMap:
+            name: rook-ceph-csi-config
+            items:
+              - key: csi-cluster-config-json
+                path: config.json
+        - name: keys-tmp-dir
+          emptyDir: {
+            medium: "Memory"
+          }
+---
+# This is a service to expose the liveness side car
+apiVersion: v1
+kind: Service
+metadata:
+  name: csi-liveness-rbdplugin-provisioner
+  labels:
+    app: csi-liveness
+spec:
+  ports:
+    - name: http-metrics
+      port: 8080
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: csi-rbdplugin-provisioner

cluster/examples/kubernetes/ceph/operator.yaml

@@ -163,11 +163,11 @@ spec:
         #- name: ROOK_CSI_REGISTRAR_IMAGE
         #  value: "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0"
         #- name: ROOK_CSI_PROVISIONER_IMAGE
-        #  value: "quay.io/k8scsi/csi-provisioner:v1.2.0"
+        #  value: "quay.io/k8scsi/csi-provisioner:v1.3.0"
         #- name: ROOK_CSI_SNAPSHOTTER_IMAGE
-        #  value: "quay.io/k8scsi/csi-snapshotter:v1.1.0"
+        #  value: "quay.io/k8scsi/csi-snapshotter:v1.2.0"
         #- name: ROOK_CSI_ATTACHER_IMAGE
-        #  value: "quay.io/k8scsi/csi-attacher:v1.1.1"
+        #  value: "quay.io/k8scsi/csi-attacher:v1.2.0"
 
         # The name of the node to pass with the downward API
         - name: NODE_NAME

cmd/rook/ceph/operator.go

@@ -59,10 +59,12 @@ func init() {
 
 	// csi deployment templates
 	operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template")
-	operatorCmd.Flags().StringVar(&csi.RBDProvisionerTemplatePath, "csi-rbd-provisioner-template-path", csi.DefaultRBDProvisionerTemplatePath, "path to ceph-csi rbd provisioner template")
+	operatorCmd.Flags().StringVar(&csi.RBDProvisionerSTSTemplatePath, "csi-rbd-provisioner-sts-template-path", csi.DefaultRBDProvisionerSTSTemplatePath, "path to ceph-csi rbd provisioner statefulset template")
+	operatorCmd.Flags().StringVar(&csi.RBDProvisionerDepTemplatePath, "csi-rbd-provisioner-dep-template-path", csi.DefaultRBDProvisionerDepTemplatePath, "path to ceph-csi rbd provisioner deployment template")
 
 	operatorCmd.Flags().StringVar(&csi.CephFSPluginTemplatePath, "csi-cephfs-plugin-template-path", csi.DefaultCephFSPluginTemplatePath, "path to ceph-csi cephfs plugin template")
-	operatorCmd.Flags().StringVar(&csi.CephFSProvisionerTemplatePath, "csi-cephfs-provisioner-template-path", csi.DefaultCephFSProvisionerTemplatePath, "path to ceph-csi cephfs provisioner template")
+	operatorCmd.Flags().StringVar(&csi.CephFSProvisionerSTSTemplatePath, "csi-cephfs-provisioner-sts-template-path", csi.DefaultCephFSProvisionerSTSTemplatePath, "path to ceph-csi cephfs provisioner statefulset template")
+	operatorCmd.Flags().StringVar(&csi.CephFSProvisionerDepTemplatePath, "csi-cephfs-provisioner-dep-template-path", csi.DefaultCephFSProvisionerDepTemplatePath, "path to ceph-csi cephfs provisioner deployment template")
 
 	flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix)
 	flags.SetLoggingFlags(operatorCmd.Flags())

pkg/operator/ceph/csi/spec.go

@@ -23,6 +23,7 @@ import (
 	"github.com/rook/rook/pkg/operator/k8sutil"
 
 	apps "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -48,11 +49,13 @@ var (
 	EnableCephFS = false
 
 	// template paths
-	RBDPluginTemplatePath      string
-	RBDProvisionerTemplatePath string
+	RBDPluginTemplatePath         string
+	RBDProvisionerSTSTemplatePath string
+	RBDProvisionerDepTemplatePath string
 
-	CephFSPluginTemplatePath      string
-	CephFSProvisionerTemplatePath string
+	CephFSPluginTemplatePath         string
+	CephFSProvisionerSTSTemplatePath string
+	CephFSProvisionerDepTemplatePath string
 
 	// configuration map for csi
 	ConfigName = "rook-ceph-csi-config"
@@ -60,21 +63,24 @@ var (
 )
 
 const (
-	KubeMinMajor = "1"
-	KubeMinMinor = "13"
+	KubeMinMajor              = "1"
+	KubeMinMinor              = "13"
+	provDeploymentSuppVersion = "14"
 
 	// image names
 	DefaultCSIPluginImage   = "quay.io/cephcsi/cephcsi:canary"
 	DefaultRegistrarImage   = "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0"
-	DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.2.0"
-	DefaultAttacherImage    = "quay.io/k8scsi/csi-attacher:v1.1.1"
-	DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.1.0"
+	DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.3.0"
+	DefaultAttacherImage    = "quay.io/k8scsi/csi-attacher:v1.2.0"
+	DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.2.0"
 
 	// template
-	DefaultRBDPluginTemplatePath         = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml"
-	DefaultRBDProvisionerTemplatePath    = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml"
-	DefaultCephFSPluginTemplatePath      = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml"
-	DefaultCephFSProvisionerTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner.yaml"
+	DefaultRBDPluginTemplatePath            = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml"
+	DefaultRBDProvisionerSTSTemplatePath    = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner-sts.yaml"
+	DefaultRBDProvisionerDepTemplatePath    = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner-dep.yaml"
+	DefaultCephFSPluginTemplatePath         = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml"
+	DefaultCephFSProvisionerSTSTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-sts.yaml"
+	DefaultCephFSProvisionerDepTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-dep.yaml"
 )
 
 func CSIEnabled() bool {
@@ -100,7 +106,7 @@ func ValidateCSIParam() error {
 		if len(RBDPluginTemplatePath) == 0 {
 			return errors.New("missing rbd plugin template path")
 		}
-		if len(RBDProvisionerTemplatePath) == 0 {
+		if len(RBDProvisionerSTSTemplatePath) == 0 && len(RBDProvisionerDepTemplatePath) == 0 {
 			return errors.New("missing rbd provisioner template path")
 		}
 	}
@@ -109,18 +115,20 @@ func ValidateCSIParam() error {
 		if len(CephFSPluginTemplatePath) == 0 {
 			return fmt.Errorf("missing cephfs plugin template path")
 		}
-		if len(CephFSProvisionerTemplatePath) == 0 {
+		if len(CephFSProvisionerSTSTemplatePath) == 0 && len(CephFSProvisionerDepTemplatePath) == 0 {
 			return errors.New("missing ceph provisioner template path")
 		}
 	}
 	return nil
 }
 
-func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
+func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *version.Info) error {
 	var (
-		err                               error
-		rbdPlugin, cephfsPlugin           *apps.DaemonSet
-		rbdProvisioner, cephfsProvisioner *apps.StatefulSet
+		err                                                   error
+		rbdPlugin, cephfsPlugin                               *apps.DaemonSet
+		rbdProvisionerSTS, cephfsProvisionerSTS               *apps.StatefulSet
+		rbdProvisionerDeployment, cephfsProvisionerDeployment *apps.Deployment
+		deployProvSTS                                         bool
 	)
 
 	// create an empty config map. config map will be filled with data
@@ -138,25 +146,43 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
 	if tp.DriverNamePrefix == "" {
 		tp.DriverNamePrefix = fmt.Sprintf("%s.", namespace)
 	}
+	if ver.Minor < provDeploymentSuppVersion {
+		deployProvSTS = true
+	}
 
 	if EnableRBD {
 		rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath, tp)
 		if err != nil {
 			return fmt.Errorf("failed to load rbd plugin template: %v", err)
 		}
-		rbdProvisioner, err = templateToStatefulSet("rbd-provisioner", RBDProvisionerTemplatePath, tp)
-		if err != nil {
-			return fmt.Errorf("failed to load rbd provisioner template: %v", err)
+		if deployProvSTS {
+			rbdProvisionerSTS, err = templateToStatefulSet("rbd-provisioner", RBDProvisionerSTSTemplatePath, tp)
+			if err != nil {
+				return fmt.Errorf("failed to load rbd provisioner statefulset template: %v", err)
+			}
+		} else {
+			rbdProvisionerDeployment, err = templateToDeployment("rbd-provisioner", RBDProvisionerDepTemplatePath, tp)
+			if err != nil {
+				return fmt.Errorf("failed to load rbd provisioner deployment template: %v", err)
+			}
 		}
+
 	}
 	if EnableCephFS {
 		cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath, tp)
 		if err != nil {
 			return fmt.Errorf("failed to load CephFS plugin template: %v", err)
 		}
-		cephfsProvisioner, err = templateToStatefulSet("cephfs-provisioner", CephFSProvisionerTemplatePath, tp)
-		if err != nil {
-			return fmt.Errorf("failed to load CephFS provisioner template: %v", err)
+		if deployProvSTS {
+			cephfsProvisionerSTS, err = templateToStatefulSet("cephfs-provisioner", CephFSProvisionerSTSTemplatePath, tp)
+			if err != nil {
+				return fmt.Errorf("failed to load CephFS provisioner statefulset template: %v", err)
+			}
+		} else {
+			cephfsProvisionerDeployment, err = templateToDeployment("cephfs-provisioner", CephFSProvisionerDepTemplatePath, tp)
+			if err != nil {
+				return fmt.Errorf("failed to load rbd provisioner deployment template: %v", err)
+			}
 		}
 	}
 
@@ -166,12 +192,16 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
 			return fmt.Errorf("failed to start rbdplugin daemonset: %v\n%v", err, rbdPlugin)
 		}
 	}
-	if rbdProvisioner != nil {
-		_, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisioner)
+	if rbdProvisionerSTS != nil {
+		_, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisionerSTS)
 		if err != nil {
-			return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisioner)
+			return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisionerSTS)
+		}
+	} else if rbdProvisionerDeployment != nil {
+		err = k8sutil.CreateDeployment("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisionerDeployment)
+		if err != nil {
+			return fmt.Errorf("failed to start rbd provisioner deployment: %v\n%v", err, rbdProvisionerDeployment)
 		}
-
 	}
 
 	if cephfsPlugin != nil {
@@ -180,12 +210,18 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
 			return fmt.Errorf("failed to start cephfs plugin daemonset: %v\n%v", err, cephfsPlugin)
 		}
 	}
-	if cephfsProvisioner != nil {
-		_, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisioner)
+
+	if cephfsProvisionerSTS != nil {
+		_, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisionerSTS)
 		if err != nil {
-			return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisioner)
+			return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisionerSTS)
 		}
 
+	} else if cephfsProvisionerDeployment != nil {
+		err = k8sutil.CreateDeployment("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisionerDeployment)
+		if err != nil {
+			return fmt.Errorf("failed to start cephfs provisioner deployment: %v\n%v", err, cephfsProvisionerDeployment)
+		}
 	}
 	return nil
 }

pkg/operator/ceph/csi/spec_test.go

@@ -26,9 +26,11 @@ import (
 
 func TestStartCSI(t *testing.T) {
 	RBDPluginTemplatePath = "csi-rbdplugin.yaml"
-	RBDProvisionerTemplatePath = "csi-rbdplugin-provisioner.yaml"
+	RBDProvisionerSTSTemplatePath = "csi-rbdplugin-provisioner-sts.yaml"
+	RBDProvisionerDepTemplatePath = "csi-rbdplugin-provisioner-dep.yaml"
 	CephFSPluginTemplatePath = "csi-cephfsplugin.yaml"
-	CephFSProvisionerTemplatePath = "csi-cephfsplugin-provisioner.yaml"
+	CephFSProvisionerSTSTemplatePath = "csi-cephfsplugin-provisioner-sts.yaml"
+	CephFSProvisionerDepTemplatePath = "csi-cephfsplugin-provisioner-dep.yaml"
 
 	CSIParam = Param{
 		CSIPluginImage:   "image",
@@ -38,6 +40,10 @@ func TestStartCSI(t *testing.T) {
 		SnapshotterImage: "image",
 	}
 	clientset := test.New(3)
-	err := StartCSIDrivers("ns", clientset)
+	serverVersion, err := clientset.Discovery().ServerVersion()
+	if err != nil {
+		assert.Nil(t, err)
+	}
+	err = StartCSIDrivers("ns", clientset, serverVersion)
 	assert.Nil(t, err)
 }

pkg/operator/ceph/csi/util.go

@@ -70,3 +70,17 @@ func templateToDaemonSet(name, templatePath string, p templateParam) (*apps.Daem
 	}
 	return &ds, nil
 }
+
+func templateToDeployment(name, templatePath string, p templateParam) (*apps.Deployment, error) {
+	var ds apps.Deployment
+	t, err := loadTemplate(name, templatePath, p)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal([]byte(t), &ds)
+	if err != nil {
+		return nil, err
+	}
+	return &ds, nil
+}

pkg/operator/ceph/operator.go

@@ -196,7 +196,7 @@ func (o *Operator) startSystemDaemons(externalCeph bool) error {
 	if err = csi.ValidateCSIParam(); err != nil {
 		return fmt.Errorf("invalid csi params: %v", err)
 	}
-	if err = csi.StartCSIDrivers(namespace, o.context.Clientset); err != nil {
+	if err = csi.StartCSIDrivers(namespace, o.context.Clientset, serverVersion); err != nil {
 		return fmt.Errorf("failed to start Ceph csi drivers: %v", err)
 	}
 	logger.Infof("successfully started Ceph CSI driver(s)")

pkg/operator/k8sutil/deployment.go

@@ -25,6 +25,7 @@ import (
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
@@ -213,3 +214,16 @@ func AddLabelToJob(key, value string, b *batchv1.Job) {
 func addLabel(key, value string, labels map[string]string) {
 	labels[key] = value
 }
+
+func CreateDeployment(name, namespace, appName string, clientset kubernetes.Interface, dep *apps.Deployment) error {
+	_, err := clientset.AppsV1().Deployments(namespace).Create(dep)
+	if err != nil {
+		if k8serrors.IsAlreadyExists(err) {
+			_, err = clientset.AppsV1().Deployments(namespace).Update(dep)
+		}
+		if err != nil {
+			return fmt.Errorf("failed to start %s deployment: %v\n%v", name, err, dep)
+		}
+	}
+	return err
+}

tests/framework/installer/ceph_manifests.go

@@ -506,6 +506,7 @@ rules:
   resources:
   - daemonsets
   - statefulsets
+  - deployments
   verbs:
   - get
   - list
@@ -905,7 +906,10 @@ rules:
     verbs: ["get", "list", "watch"]
   - apiGroups: ["apiextensions.k8s.io"]
     resources: ["customresourcedefinitions"]
-    verbs: ["create"]
+    verbs: ["create", "list", "watch", "delete", "get", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["get", "list", "watch"]
@@ -939,6 +943,9 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -1078,6 +1085,9 @@ rules:
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1