From 78fb35a93dedf87d5a308aec6b7485b3e098bbfb Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 23 Aug 2019 14:40:48 +0530 Subject: [PATCH] Use Deployment with leader election instead of StatefulSet Deployment behaves better when a node gets disconnected from the rest of the cluster - new provisioner leader is elected in ~15 seconds, while it may take up to 5 minutes for StatefulSet to start a new replica. if kube version is 1.13.x deploy provisioner as statefulset. if kube version is higher than 1.14+ deploy provisioner as deployment. Refer: kubernetes-csi/external-provisioner@52d1fbc Refer: ceph/ceph-csi#497 Signed-off-by: Madhu Rajanna --- Gopkg.lock | 1 + cluster/examples/kubernetes/ceph/common.yaml | 12 +- .../csi-cephfsplugin-provisioner-dep.yaml | 149 +++++++++++++++ ... => csi-cephfsplugin-provisioner-sts.yaml} | 0 .../rbd/csi-rbdplugin-provisioner-dep.yaml | 173 ++++++++++++++++++ ...aml => csi-rbdplugin-provisioner-sts.yaml} | 0 .../examples/kubernetes/ceph/operator.yaml | 6 +- cmd/rook/ceph/operator.go | 6 +- pkg/operator/ceph/csi/spec.go | 100 ++++++---- pkg/operator/ceph/csi/spec_test.go | 12 +- pkg/operator/ceph/csi/util.go | 14 ++ pkg/operator/ceph/operator.go | 2 +- pkg/operator/k8sutil/deployment.go | 14 ++ tests/framework/installer/ceph_manifests.go | 12 +- 14 files changed, 458 insertions(+), 43 deletions(-) create mode 100644 cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml rename cluster/examples/kubernetes/ceph/csi/template/cephfs/{csi-cephfsplugin-provisioner.yaml => csi-cephfsplugin-provisioner-sts.yaml} (100%) create mode 100644 cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml rename cluster/examples/kubernetes/ceph/csi/template/rbd/{csi-rbdplugin-provisioner.yaml => csi-rbdplugin-provisioner-sts.yaml} (100%) diff --git a/Gopkg.lock b/Gopkg.lock index 0b6e04b1d..9e3b32abe 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1465,6 +1465,7 @@ "k8s.io/apimachinery/pkg/util/version", "k8s.io/apimachinery/pkg/util/wait", "k8s.io/apimachinery/pkg/util/yaml", + "k8s.io/apimachinery/pkg/version", "k8s.io/apimachinery/pkg/watch", "k8s.io/apiserver/pkg/server", "k8s.io/client-go/discovery", diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 20d3c15b6..1c9c66895 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -531,6 +531,7 @@ rules: resources: - daemonsets - statefulsets + - deployments verbs: - get - list @@ -1156,6 +1157,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] # OLM: END CSI CEPHFS ROLE # OLM: BEGIN CSI CEPHFS ROLEBINDING --- @@ -1330,6 +1334,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] # OLM: END CSI RBD ROLE # OLM: BEGIN CSI RBD ROLEBINDING --- @@ -1434,7 +1441,10 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] - verbs: ["create"] + verbs: ["create", "list", "watch", "delete", "get", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] # OLM: END CSI RBD CLUSTER ROLE # OLM: BEGIN CSI RBD CLUSTER ROLEBINDING --- diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml new file mode 100644 index 000000000..46c9f75ba --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -0,0 +1,149 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-cephfsplugin-provisioner + namespace: {{ .Namespace }} +spec: + replicas: 2 + selector: + matchLabels: + app: csi-cephfsplugin-provisioner + template: + metadata: + labels: + app: csi-cephfsplugin-provisioner + spec: + serviceAccount: rook-csi-cephfs-provisioner-sa + containers: + - name: csi-attacher + image: {{ .AttacherImage }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--leader-election-type=leases" + - "--leader-election-namespace={{ .Namespace }}" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-provisioner + image: {{ .ProvisionerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=60s" + - "--retry-interval-start=500ms" + - "--enable-leader-election=true" + - "--leader-election-type=leases" + - "--leader-election-namespace={{ .Namespace }}" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + image: {{ .CSIPluginImage }} + args: + - "--nodeid=$(NODE_ID)" + - "--type=cephfs" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" + - "--metadatastorage=k8s_configmap" + - "--pidlimit=-1" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8081" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com + type: DirectoryOrCreate + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev + - name: ceph-csi-config + configMap: + name: rook-ceph-csi-config + items: + - key: csi-cluster-config-json + path: config.json + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } +--- +# This is a service to expose the liveness side car +apiVersion: v1 +kind: Service +metadata: + name: csi-liveness-cephfsplugin-provisioner + labels: + app: csi-liveness +spec: + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8081 + selector: + app: csi-cephfsplugin-provisioner diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml similarity index 100% rename from cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml rename to cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml new file mode 100644 index 000000000..58f091359 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -0,0 +1,173 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-rbdplugin-provisioner + namespace: {{ .Namespace }} +spec: + replicas: 2 + selector: + matchLabels: + app: csi-rbdplugin-provisioner + template: + metadata: + labels: + app: csi-rbdplugin-provisioner + spec: + serviceAccount: rook-csi-rbd-provisioner-sa + containers: + - name: csi-provisioner + image: {{ .ProvisionerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=60s" + - "--retry-interval-start=500ms" + - "--enable-leader-election=true" + - "--leader-election-type=leases" + - "--leader-election-namespace={{ .Namespace }}" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-rbdplugin-attacher + image: {{ .AttacherImage }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--leader-election-type=leases" + - "--leader-election-namespace={{ .Namespace }}" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-snapshotter + image: {{ .SnapshotterImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=60s" + - "--leader-election=true" + - "--leader-election-namespace={{ .Namespace }}" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: Always + securityContext: + privileged: true + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + image: {{ .CSIPluginImage }} + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--type=rbd" + - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" + - "--containerized=true" + - "--pidlimit=-1" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8080" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" + volumes: + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com + type: DirectoryOrCreate + - name: ceph-csi-config + configMap: + name: rook-ceph-csi-config + items: + - key: csi-cluster-config-json + path: config.json + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } +--- +# This is a service to expose the liveness side car +apiVersion: v1 +kind: Service +metadata: + name: csi-liveness-rbdplugin-provisioner + labels: + app: csi-liveness +spec: + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: csi-rbdplugin-provisioner diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml similarity index 100% rename from cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml rename to cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 85b73bef0..9f8d78858 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -163,11 +163,11 @@ spec: #- name: ROOK_CSI_REGISTRAR_IMAGE # value: "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0" #- name: ROOK_CSI_PROVISIONER_IMAGE - # value: "quay.io/k8scsi/csi-provisioner:v1.2.0" + # value: "quay.io/k8scsi/csi-provisioner:v1.3.0" #- name: ROOK_CSI_SNAPSHOTTER_IMAGE - # value: "quay.io/k8scsi/csi-snapshotter:v1.1.0" + # value: "quay.io/k8scsi/csi-snapshotter:v1.2.0" #- name: ROOK_CSI_ATTACHER_IMAGE - # value: "quay.io/k8scsi/csi-attacher:v1.1.1" + # value: "quay.io/k8scsi/csi-attacher:v1.2.0" # The name of the node to pass with the downward API - name: NODE_NAME diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index d55e7254e..81a5d3112 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -59,10 +59,12 @@ func init() { // csi deployment templates operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template") - operatorCmd.Flags().StringVar(&csi.RBDProvisionerTemplatePath, "csi-rbd-provisioner-template-path", csi.DefaultRBDProvisionerTemplatePath, "path to ceph-csi rbd provisioner template") + operatorCmd.Flags().StringVar(&csi.RBDProvisionerSTSTemplatePath, "csi-rbd-provisioner-sts-template-path", csi.DefaultRBDProvisionerSTSTemplatePath, "path to ceph-csi rbd provisioner statefulset template") + operatorCmd.Flags().StringVar(&csi.RBDProvisionerDepTemplatePath, "csi-rbd-provisioner-dep-template-path", csi.DefaultRBDProvisionerDepTemplatePath, "path to ceph-csi rbd provisioner deployment template") operatorCmd.Flags().StringVar(&csi.CephFSPluginTemplatePath, "csi-cephfs-plugin-template-path", csi.DefaultCephFSPluginTemplatePath, "path to ceph-csi cephfs plugin template") - operatorCmd.Flags().StringVar(&csi.CephFSProvisionerTemplatePath, "csi-cephfs-provisioner-template-path", csi.DefaultCephFSProvisionerTemplatePath, "path to ceph-csi cephfs provisioner template") + operatorCmd.Flags().StringVar(&csi.CephFSProvisionerSTSTemplatePath, "csi-cephfs-provisioner-sts-template-path", csi.DefaultCephFSProvisionerSTSTemplatePath, "path to ceph-csi cephfs provisioner statefulset template") + operatorCmd.Flags().StringVar(&csi.CephFSProvisionerDepTemplatePath, "csi-cephfs-provisioner-dep-template-path", csi.DefaultCephFSProvisionerDepTemplatePath, "path to ceph-csi cephfs provisioner deployment template") flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix) flags.SetLoggingFlags(operatorCmd.Flags()) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 7abfb8cae..6aa4dbfcb 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -23,6 +23,7 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/kubernetes" ) @@ -48,11 +49,13 @@ var ( EnableCephFS = false // template paths - RBDPluginTemplatePath string - RBDProvisionerTemplatePath string + RBDPluginTemplatePath string + RBDProvisionerSTSTemplatePath string + RBDProvisionerDepTemplatePath string - CephFSPluginTemplatePath string - CephFSProvisionerTemplatePath string + CephFSPluginTemplatePath string + CephFSProvisionerSTSTemplatePath string + CephFSProvisionerDepTemplatePath string // configuration map for csi ConfigName = "rook-ceph-csi-config" @@ -60,21 +63,24 @@ var ( ) const ( - KubeMinMajor = "1" - KubeMinMinor = "13" + KubeMinMajor = "1" + KubeMinMinor = "13" + provDeploymentSuppVersion = "14" // image names DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:canary" DefaultRegistrarImage = "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0" - DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.2.0" - DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v1.1.1" - DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.1.0" + DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.3.0" + DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v1.2.0" + DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.2.0" // template - DefaultRBDPluginTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml" - DefaultRBDProvisionerTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml" - DefaultCephFSPluginTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml" - DefaultCephFSProvisionerTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner.yaml" + DefaultRBDPluginTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml" + DefaultRBDProvisionerSTSTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner-sts.yaml" + DefaultRBDProvisionerDepTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner-dep.yaml" + DefaultCephFSPluginTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml" + DefaultCephFSProvisionerSTSTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-sts.yaml" + DefaultCephFSProvisionerDepTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-dep.yaml" ) func CSIEnabled() bool { @@ -100,7 +106,7 @@ func ValidateCSIParam() error { if len(RBDPluginTemplatePath) == 0 { return errors.New("missing rbd plugin template path") } - if len(RBDProvisionerTemplatePath) == 0 { + if len(RBDProvisionerSTSTemplatePath) == 0 && len(RBDProvisionerDepTemplatePath) == 0 { return errors.New("missing rbd provisioner template path") } } @@ -109,18 +115,20 @@ func ValidateCSIParam() error { if len(CephFSPluginTemplatePath) == 0 { return fmt.Errorf("missing cephfs plugin template path") } - if len(CephFSProvisionerTemplatePath) == 0 { + if len(CephFSProvisionerSTSTemplatePath) == 0 && len(CephFSProvisionerDepTemplatePath) == 0 { return errors.New("missing ceph provisioner template path") } } return nil } -func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error { +func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *version.Info) error { var ( - err error - rbdPlugin, cephfsPlugin *apps.DaemonSet - rbdProvisioner, cephfsProvisioner *apps.StatefulSet + err error + rbdPlugin, cephfsPlugin *apps.DaemonSet + rbdProvisionerSTS, cephfsProvisionerSTS *apps.StatefulSet + rbdProvisionerDeployment, cephfsProvisionerDeployment *apps.Deployment + deployProvSTS bool ) // create an empty config map. config map will be filled with data @@ -138,25 +146,43 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error { if tp.DriverNamePrefix == "" { tp.DriverNamePrefix = fmt.Sprintf("%s.", namespace) } + if ver.Minor < provDeploymentSuppVersion { + deployProvSTS = true + } if EnableRBD { rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath, tp) if err != nil { return fmt.Errorf("failed to load rbd plugin template: %v", err) } - rbdProvisioner, err = templateToStatefulSet("rbd-provisioner", RBDProvisionerTemplatePath, tp) - if err != nil { - return fmt.Errorf("failed to load rbd provisioner template: %v", err) + if deployProvSTS { + rbdProvisionerSTS, err = templateToStatefulSet("rbd-provisioner", RBDProvisionerSTSTemplatePath, tp) + if err != nil { + return fmt.Errorf("failed to load rbd provisioner statefulset template: %v", err) + } + } else { + rbdProvisionerDeployment, err = templateToDeployment("rbd-provisioner", RBDProvisionerDepTemplatePath, tp) + if err != nil { + return fmt.Errorf("failed to load rbd provisioner deployment template: %v", err) + } } + } if EnableCephFS { cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath, tp) if err != nil { return fmt.Errorf("failed to load CephFS plugin template: %v", err) } - cephfsProvisioner, err = templateToStatefulSet("cephfs-provisioner", CephFSProvisionerTemplatePath, tp) - if err != nil { - return fmt.Errorf("failed to load CephFS provisioner template: %v", err) + if deployProvSTS { + cephfsProvisionerSTS, err = templateToStatefulSet("cephfs-provisioner", CephFSProvisionerSTSTemplatePath, tp) + if err != nil { + return fmt.Errorf("failed to load CephFS provisioner statefulset template: %v", err) + } + } else { + cephfsProvisionerDeployment, err = templateToDeployment("cephfs-provisioner", CephFSProvisionerDepTemplatePath, tp) + if err != nil { + return fmt.Errorf("failed to load rbd provisioner deployment template: %v", err) + } } } @@ -166,12 +192,16 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error { return fmt.Errorf("failed to start rbdplugin daemonset: %v\n%v", err, rbdPlugin) } } - if rbdProvisioner != nil { - _, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisioner) + if rbdProvisionerSTS != nil { + _, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisionerSTS) if err != nil { - return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisioner) + return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisionerSTS) + } + } else if rbdProvisionerDeployment != nil { + err = k8sutil.CreateDeployment("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisionerDeployment) + if err != nil { + return fmt.Errorf("failed to start rbd provisioner deployment: %v\n%v", err, rbdProvisionerDeployment) } - } if cephfsPlugin != nil { @@ -180,12 +210,18 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error { return fmt.Errorf("failed to start cephfs plugin daemonset: %v\n%v", err, cephfsPlugin) } } - if cephfsProvisioner != nil { - _, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisioner) + + if cephfsProvisionerSTS != nil { + _, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisionerSTS) if err != nil { - return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisioner) + return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisionerSTS) } + } else if cephfsProvisionerDeployment != nil { + err = k8sutil.CreateDeployment("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisionerDeployment) + if err != nil { + return fmt.Errorf("failed to start cephfs provisioner deployment: %v\n%v", err, cephfsProvisionerDeployment) + } } return nil } diff --git a/pkg/operator/ceph/csi/spec_test.go b/pkg/operator/ceph/csi/spec_test.go index d68c3e2b3..d9894b20e 100644 --- a/pkg/operator/ceph/csi/spec_test.go +++ b/pkg/operator/ceph/csi/spec_test.go @@ -26,9 +26,11 @@ import ( func TestStartCSI(t *testing.T) { RBDPluginTemplatePath = "csi-rbdplugin.yaml" - RBDProvisionerTemplatePath = "csi-rbdplugin-provisioner.yaml" + RBDProvisionerSTSTemplatePath = "csi-rbdplugin-provisioner-sts.yaml" + RBDProvisionerDepTemplatePath = "csi-rbdplugin-provisioner-dep.yaml" CephFSPluginTemplatePath = "csi-cephfsplugin.yaml" - CephFSProvisionerTemplatePath = "csi-cephfsplugin-provisioner.yaml" + CephFSProvisionerSTSTemplatePath = "csi-cephfsplugin-provisioner-sts.yaml" + CephFSProvisionerDepTemplatePath = "csi-cephfsplugin-provisioner-dep.yaml" CSIParam = Param{ CSIPluginImage: "image", @@ -38,6 +40,10 @@ func TestStartCSI(t *testing.T) { SnapshotterImage: "image", } clientset := test.New(3) - err := StartCSIDrivers("ns", clientset) + serverVersion, err := clientset.Discovery().ServerVersion() + if err != nil { + assert.Nil(t, err) + } + err = StartCSIDrivers("ns", clientset, serverVersion) assert.Nil(t, err) } diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 5f84e71f0..c7e59ee54 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -70,3 +70,17 @@ func templateToDaemonSet(name, templatePath string, p templateParam) (*apps.Daem } return &ds, nil } + +func templateToDeployment(name, templatePath string, p templateParam) (*apps.Deployment, error) { + var ds apps.Deployment + t, err := loadTemplate(name, templatePath, p) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal([]byte(t), &ds) + if err != nil { + return nil, err + } + return &ds, nil +} diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index df3e44997..036612159 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -196,7 +196,7 @@ func (o *Operator) startSystemDaemons(externalCeph bool) error { if err = csi.ValidateCSIParam(); err != nil { return fmt.Errorf("invalid csi params: %v", err) } - if err = csi.StartCSIDrivers(namespace, o.context.Clientset); err != nil { + if err = csi.StartCSIDrivers(namespace, o.context.Clientset, serverVersion); err != nil { return fmt.Errorf("failed to start Ceph csi drivers: %v", err) } logger.Infof("successfully started Ceph CSI driver(s)") diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index a35596823..e9953de1f 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -25,6 +25,7 @@ import ( apps "k8s.io/api/apps/v1" v1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -213,3 +214,16 @@ func AddLabelToJob(key, value string, b *batchv1.Job) { func addLabel(key, value string, labels map[string]string) { labels[key] = value } + +func CreateDeployment(name, namespace, appName string, clientset kubernetes.Interface, dep *apps.Deployment) error { + _, err := clientset.AppsV1().Deployments(namespace).Create(dep) + if err != nil { + if k8serrors.IsAlreadyExists(err) { + _, err = clientset.AppsV1().Deployments(namespace).Update(dep) + } + if err != nil { + return fmt.Errorf("failed to start %s deployment: %v\n%v", name, err, dep) + } + } + return err +} diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index d2efad23d..8c9448b6d 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -506,6 +506,7 @@ rules: resources: - daemonsets - statefulsets + - deployments verbs: - get - list @@ -905,7 +906,10 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] - verbs: ["create"] + verbs: ["create", "list", "watch", "delete", "get", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] @@ -939,6 +943,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -1078,6 +1085,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1