From 6461d4866f2543d01b9a17c9b301dd31d56308d0 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 18 Mar 2019 12:44:26 +0530 Subject: [PATCH 1/4] update driver name as per CSI spec as per CSI spec,The driver name MUST follow domain name notation format (https://tools.ietf.org/html/rfc1035#section-2.3.1). Signed-off-by: Madhu Rajanna --- .../kubernetes/ceph/csi/example/cephfs/storageclass.yaml | 2 +- .../kubernetes/ceph/csi/example/rbd/snapshotclass.yaml | 2 +- .../kubernetes/ceph/csi/example/rbd/storageclass.yaml | 3 +-- .../csi/template/cephfs/csi-cephfsplugin-provisioner.yaml | 4 ++-- .../ceph/csi/template/cephfs/csi-cephfsplugin.yaml | 6 +++--- .../ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml | 2 +- .../ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml | 4 ++-- .../kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml | 6 +++--- 8 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml b/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml index c23814322..4d3246ca8 100644 --- a/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml @@ -2,7 +2,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: csi-cephfs -provisioner: csi-cephfsplugin +provisioner: cephfs.csi.ceph.com parameters: # Comma separated list of Ceph monitors # if using FQDN, make sure csi plugin's dns policy is appropriate. diff --git a/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml b/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml index 778f2b084..5e3b332f4 100644 --- a/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml @@ -3,7 +3,7 @@ apiVersion: snapshot.storage.k8s.io/v1alpha1 kind: VolumeSnapshotClass metadata: name: csi-rbdplugin-snapclass -snapshotter: csi-rbdplugin +snapshotter: rbd.csi.ceph.com parameters: pool: rbd monitors: mon1:port,mon2:port,... diff --git a/cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml b/cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml index d1a568ce7..cee54cb81 100644 --- a/cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml @@ -2,7 +2,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: csi-rbd -provisioner: csi-rbdplugin +provisioner: rbd.csi.ceph.com parameters: # Comma separated list of Ceph monitors # if using FQDN, make sure csi plugin's dns policy is appropriate. @@ -12,7 +12,6 @@ parameters: # secret as admin/user credentials. "monValueFromSecret" provides the # key in the secret whose value is the mons #monValueFromSecret: "monitors" - # Ceph pool into which the RBD image shall be created pool: rbd diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml index e37bd6d9c..93b517aa4 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml @@ -38,7 +38,7 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-cephfsplugin" + - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" env: - name: NODE_ID @@ -65,7 +65,7 @@ spec: volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-cephfsplugin + path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com type: DirectoryOrCreate - name: host-sys hostPath: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 9b121933a..7cb97a6c3 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -23,7 +23,7 @@ spec: args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock" lifecycle: preStop: exec: @@ -49,7 +49,7 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-cephfsplugin" + - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" env: - name: NODE_ID @@ -82,7 +82,7 @@ spec: volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/csi-cephfsplugin/ + path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/ type: DirectoryOrCreate - name: csi-plugins-dir hostPath: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml index b4de4dfde..7e995283c 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml @@ -31,5 +31,5 @@ spec: volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml index ac167e750..31b775dc0 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml @@ -53,7 +53,7 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-rbdplugin" + - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--metadatastorage=k8s_configmap" env: @@ -97,5 +97,5 @@ spec: path: /lib/modules - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index eaecaca3d..150757e19 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -24,7 +24,7 @@ spec: args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock" lifecycle: preStop: exec: @@ -50,7 +50,7 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-rbdplugin" + - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--metadatastorage=k8s_configmap" env: @@ -88,7 +88,7 @@ spec: volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate - name: plugin-mount-dir hostPath: From d99527382d201c369c8379c763fcf9cbef6d5ac7 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 18 Mar 2019 12:48:34 +0530 Subject: [PATCH 2/4] replace fmt.Errorf with errors.New if error formatting is not required we can we errors.New() Signed-off-by: Madhu Rajanna --- pkg/operator/ceph/csi/spec.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 07bf9701d..344330de0 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -17,6 +17,7 @@ limitations under the License. package csi import ( + "errors" "fmt" "github.com/rook/rook/pkg/operator/k8sutil" @@ -87,43 +88,43 @@ func SetCSINamespace(namespace string) { func ValidateCSIParam() error { if EnableRBD { if len(CSIParam.RBDPluginImage) == 0 { - return fmt.Errorf("missing csi rbd plugin image") + return errors.New("missing csi rbd plugin image") } if len(CSIParam.RegistrarImage) == 0 { - return fmt.Errorf("missing csi registrar image") + return errors.New("missing csi registrar image") } if len(CSIParam.ProvisionerImage) == 0 { - return fmt.Errorf("missing csi provisioner image") + return errors.New("missing csi provisioner image") } if len(CSIParam.AttacherImage) == 0 { - return fmt.Errorf("missing csi attacher image") + return errors.New("missing csi attacher image") } if len(RBDPluginTemplatePath) == 0 { - return fmt.Errorf("missing rbd plugin template path") + return errors.New("missing rbd plugin template path") } if len(RBDProvisionerTemplatePath) == 0 { - return fmt.Errorf("missing rbd provisioner template path") + return errors.New("missing rbd provisioner template path") } if len(RBDAttacherTemplatePath) == 0 { - return fmt.Errorf("missing rbd attacher template path") + return errors.New("missing rbd attacher template path") } } if EnableCephFS { if len(CSIParam.CephFSPluginImage) == 0 { - return fmt.Errorf("missing csi cephfs plugin image") + return errors.New("missing csi cephfs plugin image") } if len(CSIParam.RegistrarImage) == 0 { - return fmt.Errorf("missing csi registrar image") + return errors.New("missing csi registrar image") } if len(CSIParam.ProvisionerImage) == 0 { - return fmt.Errorf("missing csi provisioner image") + return errors.New("missing csi provisioner image") } if len(CephFSPluginTemplatePath) == 0 { return fmt.Errorf("missing cephfs plugin template path") } if len(CephFSProvisionerTemplatePath) == 0 { - return fmt.Errorf("missing ceph provisioner template path") + return errors.New("missing ceph provisioner template path") } } return nil From 38527239c4ffa6b8706412663b0db890d438aa00 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 18 Mar 2019 12:55:25 +0530 Subject: [PATCH 3/4] simplify socker directory name inside container socket directory name need not follow the driver name format,renamed socket directory to simple names. Signed-off-by: Madhu Rajanna --- .../cephfs/csi-cephfsplugin-provisioner.yaml | 10 ++-- .../csi/template/cephfs/csi-cephfsplugin.yaml | 6 +-- .../template/rbd/csi-rbdplugin-attacher.yaml | 4 +- .../rbd/csi-rbdplugin-provisioner.yaml | 12 ++--- .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 4 +- .../framework/installer/ceph_csi_templates.go | 54 +++++++++---------- tests/integration/ceph_csi_test.go | 6 +-- 7 files changed, 48 insertions(+), 48 deletions(-) diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml index 93b517aa4..fe26ed206 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml @@ -23,18 +23,18 @@ spec: - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /csi - name: csi-cephfsplugin securityContext: privileged: true capabilities: add: ["SYS_ADMIN"] image: {{ .CephFSPluginImage }} - args : + args: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" @@ -50,11 +50,11 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /csi - name: host-sys mountPath: /sys - name: lib-modules diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 7cb97a6c3..039acd2b6 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -45,7 +45,7 @@ spec: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true image: {{ .CephFSPluginImage }} - args : + args: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" @@ -61,11 +61,11 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock + value: unix:///csi/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /csi - name: csi-plugins-dir mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi mountPropagation: "Bidirectional" diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml index 7e995283c..fe5fb56b8 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml @@ -23,11 +23,11 @@ spec: - "--csi-address=$(ADDRESS)" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock + value: unix://csi/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi volumes: - name: socket-dir hostPath: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml index 31b775dc0..aa18a1a8d 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml @@ -23,11 +23,11 @@ spec: - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi - name: csi-snapshotter image: {{ .SnapshotterImage }} args: @@ -36,13 +36,13 @@ spec: - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + value: unix://csi/csi-provisioner.sock imagePullPolicy: Always securityContext: privileged: true volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi - name: csi-rbdplugin securityContext: privileged: true @@ -68,11 +68,11 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi - mountPath: /dev name: host-dev - mountPath: /rootfs diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index 150757e19..8c8a24baa 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -65,11 +65,11 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock + value: unix:///csi/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: plugin-dir - mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin + mountPath: /csi - name: pods-mount-dir mountPath: /var/lib/kubelet/pods mountPropagation: "Bidirectional" diff --git a/tests/framework/installer/ceph_csi_templates.go b/tests/framework/installer/ceph_csi_templates.go index 789678710..d681b7d83 100644 --- a/tests/framework/installer/ceph_csi_templates.go +++ b/tests/framework/installer/ceph_csi_templates.go @@ -43,15 +43,15 @@ const ( - "--csi-address=$(ADDRESS)" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock + value: unix:///csi/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate ` rbdProvisionerTemplate = ` @@ -80,11 +80,11 @@ const ( - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi - name: csi-snapshotter image: {{ .SnapshotterImage }} args: @@ -93,13 +93,13 @@ const ( - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: Always securityContext: privileged: true volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi - name: csi-rbdplugin securityContext: privileged: true @@ -110,7 +110,7 @@ const ( - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-rbdplugin" + - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--metadatastorage=k8s_configmap" env: @@ -125,11 +125,11 @@ const ( fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi - mountPath: /dev name: host-dev - mountPath: /rootfs @@ -154,7 +154,7 @@ const ( path: /lib/modules - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate ` rbdPluginTemplate = ` @@ -184,7 +184,7 @@ const ( args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock" lifecycle: preStop: exec: @@ -210,7 +210,7 @@ const ( - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-rbdplugin" + - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--metadatastorage=k8s_configmap" env: @@ -225,11 +225,11 @@ const ( fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock + value: unix:///csi/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: plugin-dir - mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin + mountPath: /csi - name: pods-mount-dir mountPath: /var/lib/kubelet/pods mountPropagation: "Bidirectional" @@ -248,7 +248,7 @@ const ( volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate - name: plugin-mount-dir hostPath: @@ -301,11 +301,11 @@ const ( - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /csi - name: csi-cephfsplugin securityContext: privileged: true @@ -316,7 +316,7 @@ const ( - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-cephfsplugin" + - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" env: - name: NODE_ID @@ -328,11 +328,11 @@ const ( fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /csi - name: host-sys mountPath: /sys - name: lib-modules @@ -343,7 +343,7 @@ const ( volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-cephfsplugin + path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com type: DirectoryOrCreate - name: host-sys hostPath: @@ -381,7 +381,7 @@ const ( args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock" lifecycle: preStop: exec: @@ -407,7 +407,7 @@ const ( - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-cephfsplugin" + - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" env: - name: NODE_ID @@ -419,11 +419,11 @@ const ( fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock + value: unix:///csi/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /csi - name: csi-plugins-dir mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi mountPropagation: "Bidirectional" @@ -440,7 +440,7 @@ const ( volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/csi-cephfsplugin/ + path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/ type: DirectoryOrCreate - name: csi-plugins-dir hostPath: diff --git a/tests/integration/ceph_csi_test.go b/tests/integration/ceph_csi_test.go index d7141af1f..284c42abc 100644 --- a/tests/integration/ceph_csi_test.go +++ b/tests/integration/ceph_csi_test.go @@ -26,7 +26,7 @@ import ( "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -101,7 +101,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + csiSCRBD + ` -provisioner: csi-rbdplugin +provisioner: rbd.csi.ceph.com parameters: monValueFromSecret: "monitors" pool: ` + csiPoolRBD + ` @@ -117,7 +117,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + csiSCCephFS + ` -provisioner: csi-cephFSplugin +provisioner: cephfs.csi.ceph.com parameters: monValueFromSecret: "monitors" pool: ` + csiPoolCephFS + ` From d9dd81396c37640201b7206b6acbf857a9b9382d Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 10 Apr 2019 18:27:08 +0530 Subject: [PATCH 4/4] remove attacher statefulset for rbd csi driver deploy attacher as a container inside rbd provisioner stateful set Signed-off-by: Madhu Rajanna --- Documentation/ceph-csi-drivers.md | 5 +- .../csi/rbac/cephfs/csi-provisioner-rbac.yaml | 2 +- .../ceph/csi/rbac/rbd/csi-attacher-rbac.yaml | 38 -------- .../csi/rbac/rbd/csi-provisioner-rbac.yaml | 14 ++- .../template/rbd/csi-rbdplugin-attacher.yaml | 35 -------- .../rbd/csi-rbdplugin-provisioner.yaml | 24 +++-- cmd/rook/ceph/operator.go | 1 - design/ceph-csi-driver.md | 5 +- pkg/operator/ceph/csi/spec.go | 21 +---- pkg/operator/ceph/csi/spec_test.go | 2 - .../framework/installer/ceph_csi_templates.go | 88 +++++++------------ tests/framework/installer/ceph_installer.go | 4 +- tests/framework/installer/ceph_manifests.go | 49 ++--------- 13 files changed, 79 insertions(+), 209 deletions(-) delete mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-attacher-rbac.yaml delete mode 100644 cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/ceph-csi-drivers.md index f4bdcb6bc..4a9f082ca 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/ceph-csi-drivers.md @@ -59,15 +59,13 @@ NAME READY STATUS RESTARTS AGE pod/csi-cephfsplugin-h5spd 2/2 Running 0 1d pod/csi-cephfsplugin-provisioner-0 2/2 Running 0 1d pod/csi-rbdplugin-4l6zg 2/2 Running 2 1d -pod/csi-rbdplugin-attacher-0 1/1 Running 0 1d -pod/csi-rbdplugin-provisioner-0 2/2 Running 2 1d +pod/csi-rbdplugin-provisioner-0 4/4 Running 2 1d pod/rook-ceph-agent-zlm84 1/1 Running 0 1d pod/rook-ceph-operator-c84954957-jdzk6 1/1 Running 0 1d pod/rook-discover-66hjp 1/1 Running 0 1d NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/csi-cephfsplugin-provisioner ClusterIP 10.0.0.107 1234/TCP 1d -service/csi-rbdplugin-attacher ClusterIP 10.0.0.109 1234/TCP 1d service/csi-rbdplugin-provisioner ClusterIP 10.0.0.56 1234/TCP 1d NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE @@ -84,7 +82,6 @@ replicaset.apps/rook-ceph-operator-c84954957 1 1 1 1d NAME DESIRED CURRENT AGE statefulset.apps/csi-cephfsplugin-provisioner 1 1 1d -statefulset.apps/csi-rbdplugin-attacher 1 1 1d statefulset.apps/csi-rbdplugin-provisioner 1 1 1d ``` diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml index 08efa5c52..5c099f7bb 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml @@ -14,7 +14,7 @@ rules: verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "update"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-attacher-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-attacher-rbac.yaml deleted file mode 100644 index 37b702445..000000000 --- a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-attacher-rbac.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-attacher-sa - namespace: rook-ceph - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-attacher-runner -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-attacher-role -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-attacher-sa - namespace: rook-ceph -roleRef: - kind: ClusterRole - name: rbd-external-attacher-runner - apiGroup: rbac.authorization.k8s.io diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml index 0912aad95..5969b5d06 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml @@ -15,10 +15,22 @@ rules: verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "update"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml deleted file mode 100644 index fe5fb56b8..000000000 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml +++ /dev/null @@ -1,35 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1beta2 -metadata: - name: csi-rbdplugin-attacher - namespace: {{ .Namespace }} -spec: - serviceName: csi-rbdplugin-attacher - replicas: 1 - selector: - matchLabels: - app: csi-rbdplugin-attacher - template: - metadata: - labels: - app: csi-rbdplugin-attacher - spec: - serviceAccount: rook-csi-rbd-attacher-sa - containers: - - name: csi-rbdplugin-attacher - image: {{ .AttacherImage }} - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: unix://csi/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - volumes: - - name: socket-dir - hostPath: - path: /var/lib/kubelet/plugins/rbd.csi.ceph.com - type: DirectoryOrCreate diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml index aa18a1a8d..1741b84eb 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml @@ -8,7 +8,7 @@ spec: replicas: 1 selector: matchLabels: - app: csi-rbdplugin-provisioner + app: csi-rbdplugin-provisioner template: metadata: labels: @@ -28,6 +28,18 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + - name: csi-rbdplugin-attacher + image: {{ .AttacherImage }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: csi-snapshotter image: {{ .SnapshotterImage }} args: @@ -36,7 +48,7 @@ spec: - "--v=5" env: - name: ADDRESS - value: unix://csi/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: Always securityContext: privileged: true @@ -58,7 +70,7 @@ spec: - "--metadatastorage=k8s_configmap" env: - name: HOST_ROOTFS - value: "/rootfs" + value: "/rootfs" - name: NODE_ID valueFrom: fieldRef: @@ -76,7 +88,7 @@ spec: - mountPath: /dev name: host-dev - mountPath: /rootfs - name: host-rootfs + name: host-rootfs - mountPath: /sys name: host-sys - mountPath: /lib/modules @@ -88,13 +100,13 @@ spec: path: /dev - name: host-rootfs hostPath: - path: / + path: / - name: host-sys hostPath: path: /sys - name: lib-modules hostPath: - path: /lib/modules + path: /lib/modules - name: socket-dir hostPath: path: /var/lib/kubelet/plugins/rbd.csi.ceph.com diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index 1dd270b70..14ec36e62 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -56,7 +56,6 @@ func init() { // csi deployment templates operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template") operatorCmd.Flags().StringVar(&csi.RBDProvisionerTemplatePath, "csi-rbd-provisioner-template-path", csi.DefaultRBDProvisionerTemplatePath, "path to ceph-csi rbd provisioner template") - operatorCmd.Flags().StringVar(&csi.RBDAttacherTemplatePath, "csi-rbd-attacher-template-path", csi.DefaultRBDAttacherTemplatePath, "path to ceph-csi rbd attacher template") operatorCmd.Flags().StringVar(&csi.CephFSPluginTemplatePath, "csi-cephfs-plugin-template-path", csi.DefaultCephFSPluginTemplatePath, "path to ceph-csi cephfs plugin template") operatorCmd.Flags().StringVar(&csi.CephFSProvisionerTemplatePath, "csi-cephfs-provisioner-template-path", csi.DefaultCephFSProvisionerTemplatePath, "path to ceph-csi cephfs provisioner template") diff --git a/design/ceph-csi-driver.md b/design/ceph-csi-driver.md index 018986527..94f25ef0e 100644 --- a/design/ceph-csi-driver.md +++ b/design/ceph-csi-driver.md @@ -36,9 +36,8 @@ Starting Kubernetes CSI driver brings up an [external-provisioner](https://githu For example, deploying a CephFS CSI driver consists of the following steps: 1. Creating a [RBAC for external provisioner](https://github.com/ceph/ceph-csi/blob/master/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml) and the [provisioner itself](https://github.com/ceph/ceph-csi/blob/master/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml). -2. Creating a [RBAC for external attacher](https://github.com/ceph/ceph-csi/blob/master/deploy/cephfs/kubernetes/csi-attacher-rbac.yaml) and [attacher itself](https://github.com/ceph/ceph-csi/blob/master/deploy/cephfs/kubernetes/csi-attacher-rbac.yaml). -3. Creating [RBAC for CSI driver](https://github.com/ceph/ceph-csi/blob/master/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml) and [driver DaemonSet](https://github.com/ceph/ceph-csi/blob/master/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml) -4. Creating Storage Classes for CSI provisioners. +2. Creating [RBAC for CSI driver](https://github.com/ceph/ceph-csi/blob/master/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml) and [driver DaemonSet](https://github.com/ceph/ceph-csi/blob/master/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml) +3. Creating Storage Classes for CSI provisioners. ## Integration Plan diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 344330de0..c7230671b 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -46,7 +46,6 @@ var ( // template paths RBDPluginTemplatePath string RBDProvisionerTemplatePath string - RBDAttacherTemplatePath string CephFSPluginTemplatePath string CephFSProvisionerTemplatePath string @@ -67,7 +66,6 @@ const ( // template DefaultRBDPluginTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml" DefaultRBDProvisionerTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml" - DefaultRBDAttacherTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-attacher.yaml" DefaultCephFSPluginTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml" DefaultCephFSProvisionerTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner.yaml" @@ -105,9 +103,6 @@ func ValidateCSIParam() error { if len(RBDProvisionerTemplatePath) == 0 { return errors.New("missing rbd provisioner template path") } - if len(RBDAttacherTemplatePath) == 0 { - return errors.New("missing rbd attacher template path") - } } if EnableCephFS { @@ -132,9 +127,9 @@ func ValidateCSIParam() error { func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error { var ( - err error - rbdPlugin, cephfsPlugin *apps.DaemonSet - rbdProvisioner, rbdAttacher, cephfsProvisioner *apps.StatefulSet + err error + rbdPlugin, cephfsPlugin *apps.DaemonSet + rbdProvisioner, cephfsProvisioner *apps.StatefulSet ) if EnableRBD { @@ -146,10 +141,6 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error { if err != nil { return fmt.Errorf("failed to load rbd provisioner template: %v", err) } - rbdAttacher, err = templateToStatefulSet("rbd-attacher", RBDAttacherTemplatePath) - if err != nil { - return fmt.Errorf("failed to load rbd attacher template: %v", err) - } } if EnableCephFS { cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath) @@ -175,12 +166,6 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error { } } - if rbdAttacher != nil { - _, err = k8sutil.CreateStatefulSet("csi rbd attacher", namespace, "csi-rbdplugin-attacher", clientset, rbdAttacher) - if err != nil { - return fmt.Errorf("failed to start rbd attacher statefulset: %v\n%v", err, rbdAttacher) - } - } if cephfsPlugin != nil { err = k8sutil.CreateDaemonSet("csi cephfs plugin", namespace, clientset, cephfsPlugin) diff --git a/pkg/operator/ceph/csi/spec_test.go b/pkg/operator/ceph/csi/spec_test.go index f7a71acba..0791ab4b7 100644 --- a/pkg/operator/ceph/csi/spec_test.go +++ b/pkg/operator/ceph/csi/spec_test.go @@ -27,8 +27,6 @@ import ( func TestStartCSI(t *testing.T) { RBDPluginTemplatePath = "csi-rbdplugin.yaml" RBDProvisionerTemplatePath = "csi-rbdplugin-provisioner.yaml" - RBDAttacherTemplatePath = "csi-rbdplugin-attacher.yaml" - CephFSPluginTemplatePath = "csi-cephfsplugin.yaml" CephFSProvisionerTemplatePath = "csi-cephfsplugin-provisioner.yaml" diff --git a/tests/framework/installer/ceph_csi_templates.go b/tests/framework/installer/ceph_csi_templates.go index d681b7d83..1884ac860 100644 --- a/tests/framework/installer/ceph_csi_templates.go +++ b/tests/framework/installer/ceph_csi_templates.go @@ -17,43 +17,6 @@ limitations under the License. package installer const ( - rbdAttacherTemplate = ` - kind: StatefulSet - apiVersion: apps/v1beta2 - metadata: - name: csi-rbdplugin-attacher - namespace: {{ .Namespace }} - spec: - serviceName: csi-rbdplugin-attacher - replicas: 1 - selector: - matchLabels: - app: csi-rbdplugin-attacher - template: - metadata: - labels: - app: csi-rbdplugin-attacher - spec: - serviceAccount: rook-csi-rbd-attacher-sa - containers: - - name: csi-rbdplugin-attacher - image: {{ .AttacherImage }} - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: unix:///csi/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - volumes: - - name: socket-dir - hostPath: - path: /var/lib/kubelet/plugins/rbd.csi.ceph.com - type: DirectoryOrCreate -` rbdProvisionerTemplate = ` kind: StatefulSet apiVersion: apps/v1beta2 @@ -65,7 +28,7 @@ const ( replicas: 1 selector: matchLabels: - app: csi-rbdplugin-provisioner + app: csi-rbdplugin-provisioner template: metadata: labels: @@ -85,6 +48,19 @@ const ( volumeMounts: - name: socket-dir mountPath: /csi + - name: csi-rbdplugin-attacher + image: {{ .AttacherImage }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-snapshotter image: {{ .SnapshotterImage }} args: @@ -115,7 +91,7 @@ const ( - "--metadatastorage=k8s_configmap" env: - name: HOST_ROOTFS - value: "/rootfs" + value: "/rootfs" - name: NODE_ID valueFrom: fieldRef: @@ -133,7 +109,7 @@ const ( - mountPath: /dev name: host-dev - mountPath: /rootfs - name: host-rootfs + name: host-rootfs - mountPath: /sys name: host-sys - mountPath: /lib/modules @@ -145,13 +121,13 @@ const ( path: /dev - name: host-rootfs hostPath: - path: / + path: / - name: host-sys hostPath: path: /sys - name: lib-modules hostPath: - path: /lib/modules + path: /lib/modules - name: socket-dir hostPath: path: /var/lib/kubelet/plugins/rbd.csi.ceph.com @@ -162,7 +138,7 @@ const ( apiVersion: apps/v1beta2 metadata: name: csi-rbdplugin - namespace: {{ .Namespace }} + namespace: {{ .Namespace }} spec: selector: matchLabels: @@ -174,10 +150,10 @@ const ( spec: serviceAccount: rook-csi-rbd-plugin-sa hostNetwork: true - hostPID: true + hostPID: true # to use e.g. Rook orchestrated cluster, and mons' FQDN is # resolved through k8s service, set dns policy to cluster first - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirstWithHostNet containers: - name: driver-registrar image: {{ .RegistrarImage }} @@ -188,7 +164,7 @@ const ( lifecycle: preStop: exec: - command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"] + command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"] env: - name: KUBE_NODE_NAME valueFrom: @@ -215,7 +191,7 @@ const ( - "--metadatastorage=k8s_configmap" env: - name: HOST_ROOTFS - value: "/rootfs" + value: "/rootfs" - name: NODE_ID valueFrom: fieldRef: @@ -239,7 +215,7 @@ const ( - mountPath: /dev name: host-dev - mountPath: /rootfs - name: host-rootfs + name: host-rootfs - mountPath: /sys name: host-sys - mountPath: /lib/modules @@ -251,7 +227,7 @@ const ( path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate - name: plugin-mount-dir - hostPath: + hostPath: path: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/ type: DirectoryOrCreate - name: registration-dir @@ -267,7 +243,7 @@ const ( path: /dev - name: host-rootfs hostPath: - path: / + path: / - name: host-sys hostPath: path: /sys @@ -280,13 +256,13 @@ const ( apiVersion: apps/v1beta2 metadata: name: csi-cephfsplugin-provisioner - namespace: {{ .Namespace }} + namespace: {{ .Namespace }} spec: serviceName: "csi-cephfsplugin-provisioner" replicas: 1 selector: matchLabels: - app: csi-cephfsplugin-provisioner + app: csi-cephfsplugin-provisioner template: metadata: labels: @@ -339,7 +315,7 @@ const ( mountPath: /lib/modules readOnly: true - name: host-dev - mountPath: /dev + mountPath: /dev volumes: - name: socket-dir hostPath: @@ -360,7 +336,7 @@ const ( apiVersion: apps/v1beta2 metadata: name: csi-cephfsplugin - namespace: {{ .Namespace }} + namespace: {{ .Namespace }} spec: selector: matchLabels: @@ -374,7 +350,7 @@ const ( hostNetwork: true # to use e.g. Rook orchestrated cluster, and mons' FQDN is # resolved through k8s service, set dns policy to cluster first - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirstWithHostNet containers: - name: driver-registrar image: {{ .RegistrarImage }} @@ -385,7 +361,7 @@ const ( lifecycle: preStop: exec: - command: ["/bin/sh", "-c", "rm -rf /registration/csi-cephfsplugin /registration/csi-cephfsplugin-reg.sock"] + command: ["/bin/sh", "-c", "rm -rf /registration/csi-cephfsplugin /registration/csi-cephfsplugin-reg.sock"] env: - name: KUBE_NODE_NAME valueFrom: diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 85ecde714..d0c354e78 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -31,7 +31,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/tests/framework/utils" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -408,8 +408,6 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-global", nil) h.k8shelper.Clientset.RbacV1beta1().Roles(systemNamespace).Delete("rook-ceph-system", nil) - h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-attacher-role", nil) - h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-external-attacher-runner", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-nodeplugin", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-csi-nodeplugin", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-provisioner-role", nil) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index f9936b9b4..da589f3b2 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -499,7 +499,6 @@ metadata: name: csi-rbd-config namespace: ` + namespace + ` data: - csi-rbdplugin-attacher.yaml: |-` + rbdAttacherTemplate + ` csi-rbdplugin-provisioner.yaml: |-` + rbdProvisionerTemplate + ` csi-rbdplugin.yaml: |-` + rbdPluginTemplate + ` --- @@ -514,44 +513,6 @@ data: --- apiVersion: v1 kind: ServiceAccount -metadata: - name: rook-csi-rbd-attacher-sa - namespace: ` + namespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-attacher-runner -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-attacher-role -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-attacher-sa - namespace: ` + namespace + ` -roleRef: - kind: ClusterRole - name: rbd-external-attacher-runner - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount metadata: name: rook-csi-rbd-plugin-sa namespace: ` + namespace + ` @@ -609,7 +570,7 @@ rules: verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "update"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] @@ -637,6 +598,12 @@ rules: - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] --- kind: ClusterRoleBinding @@ -709,7 +676,7 @@ rules: verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "update"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"]