From 71f369303e8a3bd8eda70bf74ae5879d82d8cf60 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 19 Jun 2019 12:06:59 +0530 Subject: [PATCH] update rook to use ceph-csi canary version ceph-csi canary version is close to ceph-csi v1.1.0 release Signed-off-by: Madhu Rajanna --- Documentation/ceph-csi-drivers.md | 13 ++- .../ceph/csi/example/cephfs/storageclass.yaml | 16 ++-- .../ceph/csi/example/rbd/snapshotclass.yaml | 5 +- .../csi/rbac/cephfs/csi-provisioner-rbac.yaml | 33 +++++++- .../csi/rbac/rbd/csi-nodeplugin-rbac.yaml | 3 + .../csi/rbac/rbd/csi-provisioner-rbac.yaml | 35 ++++++-- .../cephfs/csi-cephfsplugin-provisioner.yaml | 11 ++- .../csi/template/cephfs/csi-cephfsplugin.yaml | 16 +++- .../template/rbd/csi-rbdplugin-attacher.yaml | 35 -------- .../rbd/csi-rbdplugin-provisioner.yaml | 4 +- .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 4 +- .../ceph/operator-openshift-with-csi.yaml | 14 ++-- .../kubernetes/ceph/operator-with-csi.yaml | 14 ++-- cmd/rook/ceph/operator.go | 3 +- pkg/operator/ceph/csi/spec.go | 57 ++++++------- pkg/operator/ceph/csi/spec_test.go | 11 ++- pkg/operator/ceph/csi/util_test.go | 12 +-- .../framework/installer/ceph_csi_templates.go | 44 +++++++++- tests/framework/installer/ceph_manifests.go | 80 +++++++++++++++---- tests/integration/ceph_csi_test.go | 34 ++++---- 20 files changed, 276 insertions(+), 168 deletions(-) delete mode 100644 cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/ceph-csi-drivers.md index 0fdc6b7ac..d098d91fa 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/ceph-csi-drivers.md @@ -270,8 +270,12 @@ In short, as the documentation describes it: In [snapshotClass](cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml), the `csi.storage.k8s.io/snapshotter-secret-name` parameter should reference the -name of the secret created for the rbdplugin. The `monitors` are a comma -separated list of your Ceph monitors and `pool` to reflect the Ceph pool name. +name of the secret created for the rbdplugin and `pool` to reflect the Ceph pool name. + +Update the value of the `clusterID` field to match the namespace that rook is +running in. When Ceph CSI is deployed by Rook, the operator will automatically +maintain a config map whose contents will match this key. By default this is +"rook-ceph". ```console kubectl create -f cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml @@ -359,7 +363,10 @@ expect a pool named `cephfs_data` in your Ceph cluster. You can create this pool using [rook file-system CRD](https://github.com/rook/rook/blob/master/Documentation/ceph-filesystem-crd.md). -Please update `monitors` to reflect the Ceph monitors. +Update `fsName` value with CephFilesystem pool name created. + +Update the value of the `clusterID` field to match the namespace that rook is +running in. ```console kubectl create -f cluster/examples/kubernetes/ceph/csi/example/cephfs/storegeclass.yaml diff --git a/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml b/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml index 4d3246ca8..ee6b4dacb 100644 --- a/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml @@ -4,17 +4,11 @@ metadata: name: csi-cephfs provisioner: cephfs.csi.ceph.com parameters: - # Comma separated list of Ceph monitors - # if using FQDN, make sure csi plugin's dns policy is appropriate. - monitors: mon1:port,mon2:port,... + # clusterID is the namespace where operator is deployed. + clusterID: rook-ceph - # For provisionVolume: "true": - # A new volume will be created along with a new Ceph user. - # Requires admin credentials (adminID, adminKey). - # For provisionVolume: "false": - # It is assumed the volume already exists and the user is expected - # to provide path to that volume (rootPath) and user credentials (userID, userKey). - provisionVolume: "true" + # CephFS filesystem name into which the volume shall be created + fsName: myfs # Ceph pool into which the volume shall be created # Required for provisionVolume: "true" @@ -35,3 +29,5 @@ parameters: # or by setting the default mounter explicitly via --volumemounter command-line argument. # mounter: kernel reclaimPolicy: Delete +mountOptions: + - noexec diff --git a/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml b/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml index 5e3b332f4..2e24e6cbd 100644 --- a/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml @@ -6,6 +6,9 @@ metadata: snapshotter: rbd.csi.ceph.com parameters: pool: rbd - monitors: mon1:port,mon2:port,... + # Specify a string that identifies your cluster. Ceph CSI supports any + # unique string. When Ceph CSI is deployed by Rook use the Rook namespace, + # for example "rook-ceph". + clusterID: rook-ceph csi.storage.k8s.io/snapshotter-secret-name: csi-rbd-secret csi.storage.k8s.io/snapshotter-secret-namespace: default diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml index 218e81a5f..0778f0868 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -36,9 +37,6 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update"] @@ -59,3 +57,32 @@ roleRef: kind: ClusterRole name: cephfs-external-provisioner-runner apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: rook-ceph + name: cephfs-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role-cfg + namespace: rook-ceph +subjects: + - kind: ServiceAccount + name: cephfs-csi-provisioner + namespace: rook-ceph +roleRef: + kind: Role + name: cephfs-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml index d37d0cc86..5c524aaf7 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml @@ -22,6 +22,9 @@ metadata: labels: rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "update"] diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml index 1a5dd570f..985b21a0a 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml @@ -43,15 +43,9 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "create", "update"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] @@ -75,3 +69,32 @@ roleRef: kind: ClusterRole name: rbd-external-provisioner-runner apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: rook-ceph + name: rbd-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role-cfg + namespace: rook-ceph +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + namespace: rook-ceph +roleRef: + kind: Role + name: rbd-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml index daa402367..b2f7563a1 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml @@ -45,9 +45,10 @@ spec: privileged: true capabilities: add: ["SYS_ADMIN"] - image: {{ .CephFSPluginImage }} + image: {{ .CSIPluginImage }} args: - "--nodeid=$(NODE_ID)" + - "--type=cephfs" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=cephfs.csi.ceph.com" @@ -74,6 +75,8 @@ spec: readOnly: true - name: host-dev mountPath: /dev + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ volumes: - name: socket-dir hostPath: @@ -88,3 +91,9 @@ spec: - name: host-dev hostPath: path: /dev + - name: ceph-csi-config + configMap: + name: rook-ceph-mon-endpoints + items: + - key: csi-cluster-config-json + path: config.json diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 04fcbb9ee..78af8f2bb 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -44,13 +44,15 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: {{ .CephFSPluginImage }} + image: {{ .CSIPluginImage }} args: - "--nodeid=$(NODE_ID)" + - "--type=cephfs" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" + - "--mountcachedir=/mount-cache-dir" env: - name: NODE_ID valueFrom: @@ -79,6 +81,10 @@ spec: readOnly: true - name: host-dev mountPath: /dev + - name: mount-cache-dir + mountPath: /mount-cache-dir + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ volumes: - name: plugin-dir hostPath: @@ -105,3 +111,11 @@ spec: - name: host-dev hostPath: path: /dev + - name: mount-cache-dir + emptyDir: {} + - name: ceph-csi-config + configMap: + name: rook-ceph-mon-endpoints + items: + - key: csi-cluster-config-json + path: config.json diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml deleted file mode 100644 index 64145d4bc..000000000 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml +++ /dev/null @@ -1,35 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-rbdplugin-attacher - namespace: {{ .Namespace }} -spec: - serviceName: csi-rbdplugin-attacher - replicas: 1 - selector: - matchLabels: - app: csi-rbdplugin-attacher - template: - metadata: - labels: - app: csi-rbdplugin-attacher - spec: - serviceAccount: rook-csi-rbd-attacher-sa - containers: - - name: csi-rbdplugin-attacher - image: {{ .AttacherImage }} - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin - volumes: - - name: socket-dir - hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin - type: DirectoryOrCreate diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml index 842899112..7768859b6 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml @@ -60,11 +60,12 @@ spec: privileged: true capabilities: add: ["SYS_ADMIN"] - image: {{ .RBDPluginImage }} + image: {{ .CSIPluginImage }} args : - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" + - "--type=rbd" - "--drivername=rbd.csi.ceph.com" - "--containerized=true" env: @@ -118,4 +119,3 @@ spec: items: - key: csi-cluster-config-json path: config.json - diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index cdb7f135b..746c3d6e5 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -45,11 +45,12 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: {{ .RBDPluginImage }} + image: {{ .CSIPluginImage }} args : - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" + - "--type=rbd" - "--drivername=rbd.csi.ceph.com" - "--containerized=true" env: @@ -121,4 +122,3 @@ spec: items: - key: csi-cluster-config-json path: config.json - diff --git a/cluster/examples/kubernetes/ceph/operator-openshift-with-csi.yaml b/cluster/examples/kubernetes/ceph/operator-openshift-with-csi.yaml index 12ed22820..fc66fb5bc 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift-with-csi.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift-with-csi.yaml @@ -122,20 +122,18 @@ spec: # CSI enablement - name: ROOK_CSI_ENABLE_CEPHFS value: "true" - - name: ROOK_CSI_CEPHFS_IMAGE - value: "quay.io/cephcsi/cephfsplugin:canary" + - name: ROOK_CSI_CEPH_IMAGE + value: "quay.io/cephcsi/cephcsi:canary" - name: ROOK_CSI_ENABLE_RBD value: "true" - - name: ROOK_CSI_RBD_IMAGE - value: "quay.io/cephcsi/rbdplugin:canary" - name: ROOK_CSI_REGISTRAR_IMAGE - value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2" + value: "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0" - name: ROOK_CSI_PROVISIONER_IMAGE - value: "quay.io/k8scsi/csi-provisioner:v1.0.1" + value: "quay.io/k8scsi/csi-provisioner:v1.2.0" - name: ROOK_CSI_SNAPSHOTTER_IMAGE - value: "quay.io/k8scsi/csi-snapshotter:v1.0.1" + value: "quay.io/k8scsi/csi-snapshotter:v1.1.0" - name: ROOK_CSI_ATTACHER_IMAGE - value: "quay.io/k8scsi/csi-attacher:v1.0.1" + value: "quay.io/k8scsi/csi-attacher:v1.1.1" - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED value: "true" - name: FLEXVOLUME_DIR_PATH diff --git a/cluster/examples/kubernetes/ceph/operator-with-csi.yaml b/cluster/examples/kubernetes/ceph/operator-with-csi.yaml index 7f1819c9c..2ea1b7c6b 100644 --- a/cluster/examples/kubernetes/ceph/operator-with-csi.yaml +++ b/cluster/examples/kubernetes/ceph/operator-with-csi.yaml @@ -40,20 +40,18 @@ spec: # CSI enablement - name: ROOK_CSI_ENABLE_CEPHFS value: "true" - - name: ROOK_CSI_CEPHFS_IMAGE - value: "quay.io/cephcsi/cephfsplugin:canary" + - name: ROOK_CSI_CEPH_IMAGE + value: "quay.io/cephcsi/cephcsi:canary" - name: ROOK_CSI_ENABLE_RBD value: "true" - - name: ROOK_CSI_RBD_IMAGE - value: "quay.io/cephcsi/rbdplugin:canary" - name: ROOK_CSI_REGISTRAR_IMAGE - value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2" + value: "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0" - name: ROOK_CSI_PROVISIONER_IMAGE - value: "quay.io/k8scsi/csi-provisioner:v1.0.1" + value: "quay.io/k8scsi/csi-provisioner:v1.2.0" - name: ROOK_CSI_SNAPSHOTTER_IMAGE - value: "quay.io/k8scsi/csi-snapshotter:v1.0.1" + value: "quay.io/k8scsi/csi-snapshotter:v1.1.0" - name: ROOK_CSI_ATTACHER_IMAGE - value: "quay.io/k8scsi/csi-attacher:v1.0.1" + value: "quay.io/k8scsi/csi-attacher:v1.1.1" # The name of the node to pass with the downward API - name: NODE_NAME valueFrom: diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index c0455ab25..b2703d4bb 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -46,8 +46,7 @@ func init() { operatorCmd.Flags().BoolVar(&csi.EnableRBD, "csi-enable-rbd", false, "enable ceph-csi rbd support") operatorCmd.Flags().BoolVar(&csi.EnableCephFS, "csi-enable-cephfs", false, "enable ceph-csi cephfs support") // csi images - operatorCmd.Flags().StringVar(&csi.CSIParam.RBDPluginImage, "csi-rbd-image", csi.DefaultRBDPluginImage, "ceph-csi rbd plugin image") - operatorCmd.Flags().StringVar(&csi.CSIParam.CephFSPluginImage, "csi-cephfs-image", csi.DefaultCephFSPluginImage, "ceph-csi cephfs plugin image") + operatorCmd.Flags().StringVar(&csi.CSIParam.CSIPluginImage, "csi-ceph-image", csi.DefaultCSIPluginImage, "ceph-csi plugin image") operatorCmd.Flags().StringVar(&csi.CSIParam.RegistrarImage, "csi-registrar-image", csi.DefaultRegistrarImage, "csi registrar image") operatorCmd.Flags().StringVar(&csi.CSIParam.ProvisionerImage, "csi-provisioner-image", csi.DefaultProvisionerImage, "csi provisioner image") operatorCmd.Flags().StringVar(&csi.CSIParam.AttacherImage, "csi-attacher-image", csi.DefaultAttacherImage, "csi attacher image") diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 7a7b53b77..ca6233e4b 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -29,12 +29,11 @@ import ( type Param struct { Namespace string - RBDPluginImage string - CephFSPluginImage string - RegistrarImage string - ProvisionerImage string - AttacherImage string - SnapshotterImage string + CSIPluginImage string + RegistrarImage string + ProvisionerImage string + AttacherImage string + SnapshotterImage string } var ( @@ -56,12 +55,11 @@ const ( KubeMinMinor = "13" // image names - DefaultRBDPluginImage = "quay.io/cephcsi/rbdplugin:v1.0.0" - DefaultCephFSPluginImage = "quay.io/cephcsi/cephfsplugin:v1.0.0" - DefaultRegistrarImage = "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2" - DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.0.1" - DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v1.0.1" - DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.0.1" + DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:canary" + DefaultRegistrarImage = "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0" + DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.2.0" + DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v1.1.0" + DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.1.1" // template DefaultRBDPluginTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml" @@ -81,19 +79,21 @@ func SetCSINamespace(namespace string) { } func ValidateCSIParam() error { + + if len(CSIParam.CSIPluginImage) == 0 { + return errors.New("missing csi rbd plugin image") + } + if len(CSIParam.RegistrarImage) == 0 { + return errors.New("missing csi registrar image") + } + if len(CSIParam.ProvisionerImage) == 0 { + return errors.New("missing csi provisioner image") + } + if len(CSIParam.AttacherImage) == 0 { + return errors.New("missing csi attacher image") + } + if EnableRBD { - if len(CSIParam.RBDPluginImage) == 0 { - return errors.New("missing csi rbd plugin image") - } - if len(CSIParam.RegistrarImage) == 0 { - return errors.New("missing csi registrar image") - } - if len(CSIParam.ProvisionerImage) == 0 { - return errors.New("missing csi provisioner image") - } - if len(CSIParam.AttacherImage) == 0 { - return errors.New("missing csi attacher image") - } if len(RBDPluginTemplatePath) == 0 { return errors.New("missing rbd plugin template path") } @@ -103,15 +103,6 @@ func ValidateCSIParam() error { } if EnableCephFS { - if len(CSIParam.CephFSPluginImage) == 0 { - return errors.New("missing csi cephfs plugin image") - } - if len(CSIParam.RegistrarImage) == 0 { - return errors.New("missing csi registrar image") - } - if len(CSIParam.ProvisionerImage) == 0 { - return errors.New("missing csi provisioner image") - } if len(CephFSPluginTemplatePath) == 0 { return fmt.Errorf("missing cephfs plugin template path") } diff --git a/pkg/operator/ceph/csi/spec_test.go b/pkg/operator/ceph/csi/spec_test.go index 0791ab4b7..d68c3e2b3 100644 --- a/pkg/operator/ceph/csi/spec_test.go +++ b/pkg/operator/ceph/csi/spec_test.go @@ -31,12 +31,11 @@ func TestStartCSI(t *testing.T) { CephFSProvisionerTemplatePath = "csi-cephfsplugin-provisioner.yaml" CSIParam = Param{ - RBDPluginImage: "image", - CephFSPluginImage: "image", - RegistrarImage: "image", - ProvisionerImage: "image", - AttacherImage: "image", - SnapshotterImage: "image", + CSIPluginImage: "image", + RegistrarImage: "image", + ProvisionerImage: "image", + AttacherImage: "image", + SnapshotterImage: "image", } clientset := test.New(3) err := StartCSIDrivers("ns", clientset) diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go index ec9ca898e..e0fc75049 100644 --- a/pkg/operator/ceph/csi/util_test.go +++ b/pkg/operator/ceph/csi/util_test.go @@ -30,7 +30,7 @@ kind: DaemonSet apiVersion: apps/v1 metadata: name: test-label - namespace: {{ .Namespace }} + namespace: {{ .Namespace }} spec: selector: matchLabels: @@ -45,16 +45,16 @@ spec: - name: registrar image: {{ .RegistrarImage }} - name: rbdplugin - image: {{ .RBDPluginImage }} + image: {{ .CSIPluginImage }} - name: cephfsplugin - image: {{ .CephFSPluginImage }} + image: {{ .CSIPluginImage }} `) testSSTemplate = []byte(` kind: StatefulSet apiVersion: apps/v1 metadata: name: test-label - namespace: {{ .Namespace }} + namespace: {{ .Namespace }} spec: selector: matchLabels: @@ -71,9 +71,9 @@ spec: - name: provisioner image: {{ .ProvisionerImage }} - name: rbdplugin - image: {{ .RBDPluginImage }} + image: {{ .CSIPluginImage }} - name: cephfsplugin - image: {{ .CephFSPluginImage }} + image: {{ .CSIPluginImage }} `) ) diff --git a/tests/framework/installer/ceph_csi_templates.go b/tests/framework/installer/ceph_csi_templates.go index 8ec4bd907..1293d5ac3 100644 --- a/tests/framework/installer/ceph_csi_templates.go +++ b/tests/framework/installer/ceph_csi_templates.go @@ -81,11 +81,12 @@ const ( privileged: true capabilities: add: ["SYS_ADMIN"] - image: {{ .RBDPluginImage }} + image: {{ .CSIPluginImage }} args : - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" + - "--type=rbd" - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--metadatastorage=k8s_configmap" @@ -181,11 +182,12 @@ const ( capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: {{ .RBDPluginImage }} + image: {{ .CSIPluginImage }} args : - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" + - "--type=rbd" - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--metadatastorage=k8s_configmap" @@ -221,6 +223,8 @@ const ( - mountPath: /lib/modules name: lib-modules readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ volumes: - name: plugin-dir hostPath: @@ -250,6 +254,12 @@ const ( - name: lib-modules hostPath: path: /lib/modules + - name: ceph-csi-config + configMap: + name: rook-ceph-mon-endpoints + items: + - key: csi-cluster-config-json + path: config.json ` cephfsProvisionerTemplate = ` kind: StatefulSet @@ -299,11 +309,12 @@ const ( privileged: true capabilities: add: ["SYS_ADMIN"] - image: {{ .CephFSPluginImage }} + image: {{ .CSIPluginImage }} args : - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" + - "--type=cephfs" - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" env: @@ -328,6 +339,9 @@ const ( readOnly: true - name: host-dev mountPath: /dev + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + volumes: - name: socket-dir hostPath: @@ -342,6 +356,12 @@ const ( - name: host-dev hostPath: path: /dev + - name: ceph-csi-config + configMap: + name: rook-ceph-mon-endpoints + items: + - key: csi-cluster-config-json + path: config.json ` cephfsPluginTemplate = ` kind: DaemonSet @@ -390,13 +410,15 @@ const ( capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: {{ .CephFSPluginImage }} + image: {{ .CSIPluginImage }} args : - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" + - "--type=cephfs" - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" + - "--mountcachedir=/mount-cache-dir" env: - name: NODE_ID valueFrom: @@ -425,6 +447,11 @@ const ( readOnly: true - name: host-dev mountPath: /dev + - name: mount-cache-dir + mountPath: /mount-cache-dir + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + volumes: - name: plugin-dir hostPath: @@ -451,5 +478,14 @@ const ( - name: host-dev hostPath: path: /dev + - name: mount-cache-dir + emptyDir: {} + - name: ceph-csi-config + configMap: + name: rook-ceph-mon-endpoints + items: + - key: csi-cluster-config-json + path: config.json + ` ) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 9d269910d..29eb72173 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -620,6 +620,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] --- kind: ClusterRoleBinding @@ -674,15 +677,9 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "create", "update"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] @@ -713,6 +710,33 @@ roleRef: name: rbd-external-provisioner-runner apiGroup: rbac.authorization.k8s.io --- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: ` + namespace + ` + name: rbd-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role-cfg + namespace: ` + namespace + ` +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + namespace: ` + namespace + ` +roleRef: + kind: Role + name: rbd-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -804,9 +828,6 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] @@ -828,6 +849,33 @@ roleRef: name: cephfs-external-provisioner-runner apiGroup: rbac.authorization.k8s.io --- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: ` + namespace + ` + name: cephfs-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role-cfg + namespace: ` + namespace + ` +subjects: + - kind: ServiceAccount + name: cephfs-csi-provisioner + namespace: ` + namespace + ` +roleRef: + kind: Role + name: cephfs-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -875,20 +923,18 @@ spec: # CSI enablement - name: ROOK_CSI_ENABLE_CEPHFS value: "true" - - name: ROOK_CSI_CEPHFS_IMAGE - value: "quay.io/cephcsi/cephfsplugin:v1.0.0" + - name: ROOK_CSI_CEPH_IMAGE + value: "quay.io/cephcsi/cephcsi:canary" - name: ROOK_CSI_ENABLE_RBD value: "true" - - name: ROOK_CSI_RBD_IMAGE - value: "quay.io/cephcsi/rbdplugin:v1.0.0" - name: ROOK_CSI_REGISTRAR_IMAGE - value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2" + value: "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0" - name: ROOK_CSI_PROVISIONER_IMAGE - value: "quay.io/k8scsi/csi-provisioner:v1.0.1" + value: "quay.io/k8scsi/csi-provisioner:v1.2.0" - name: ROOK_CSI_SNAPSHOTTER_IMAGE - value: "quay.io/k8scsi/csi-snapshotter:v1.0.1" + value: "quay.io/k8scsi/csi-snapshotter:v1.1.0" - name: ROOK_CSI_ATTACHER_IMAGE - value: "quay.io/k8scsi/csi-attacher:v1.0.1" + value: "quay.io/k8scsi/csi-attacher:v1.1.1" volumeMounts: - mountPath: /etc/ceph-csi/rbd name: csi-rbd-config diff --git a/tests/integration/ceph_csi_test.go b/tests/integration/ceph_csi_test.go index 22fcb9208..a9fd5a5c3 100644 --- a/tests/integration/ceph_csi_test.go +++ b/tests/integration/ceph_csi_test.go @@ -17,10 +17,6 @@ limitations under the License. package integration import ( - "encoding/json" - "strings" - - monclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" @@ -52,7 +48,17 @@ func runCephCSIE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suit } func isCSIRunnig(k8sh *utils.K8sHelper, namespace string) bool { - return k8sh.IsPodWithLabelPresent("app=csi-rbdplugin", installer.SystemNamespace(namespace)) && k8sh.IsPodWithLabelPresent("app=csi-cephfsplugin", installer.SystemNamespace(namespace)) + err := k8sh.WaitForLabeledPodsToRun("app=csi-rbdplugin", installer.SystemNamespace(namespace)) + if err != nil { + logger.Errorf("rbd pods are not running,failed with error %+v", err) + return false + } + err = k8sh.WaitForLabeledPodsToRun("app=csi-cephfsplugin", installer.SystemNamespace(namespace)) + if err != nil { + logger.Errorf("cephfs pods are not running,failed with error %+v", err) + return false + } + return true } func createCephCSISecret(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) { @@ -60,17 +66,6 @@ func createCephCSISecret(helper *clients.TestClient, k8sh *utils.K8sHelper, s su keyResult, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs) logger.Infof("Ceph get-key: %s", keyResult) require.Nil(s.T(), err) - commandArgs = []string{"-c", "ceph mon_status"} - monResult, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs) - logger.Infof("Ceph mon_status: %s", monResult) - require.Nil(s.T(), err) - - var mon monclient.MonStatusResponse - err = json.Unmarshal([]byte(monResult), &mon) - require.Nil(s.T(), err) - require.True(s.T(), len(mon.MonMap.Mons) > 0, "no mon found") - monStr := strings.Split(mon.MonMap.Mons[0].Address, "/")[0] - require.True(s.T(), len(monStr) > 0, "invalid mon addr") _, err = k8sh.Clientset.CoreV1().Secrets(namespace).Create(&v1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -81,7 +76,6 @@ func createCephCSISecret(helper *clients.TestClient, k8sh *utils.K8sHelper, s su "admin": []byte(keyResult), "adminID": []byte("admin"), "adminKey": []byte(keyResult), - "monitors": []byte(monStr), }, }) require.Nil(s.T(), err) @@ -104,8 +98,8 @@ metadata: name: ` + csiSCRBD + ` provisioner: rbd.csi.ceph.com parameters: - monValueFromSecret: "monitors" pool: ` + csiPoolRBD + ` + clusterID: ` + namespace + ` csi.storage.k8s.io/provisioner-secret-name: ` + csiSecretName + ` csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` csi.storage.k8s.io/node-publish-secret-name: ` + csiSecretName + ` @@ -120,8 +114,8 @@ metadata: name: ` + csiSCCephFS + ` provisioner: cephfs.csi.ceph.com parameters: - monValueFromSecret: "monitors" - provisionVolume: "true" + clusterID: ` + namespace + ` + fsName: ` + csiPoolCephFS + ` pool: ` + csiPoolCephFS + `-data0 csi.storage.k8s.io/provisioner-secret-name: ` + csiSecretName + ` csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + `