From 3806baffe121a166a85f3d23f54571cd3f5932ae Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Thu, 24 Jan 2019 12:38:07 -0500 Subject: [PATCH] add ceph csi deploy options Signed-off-by: Huamin Chen --- Documentation/ceph-csi-drivers.md | 192 +++---- PendingReleaseNotes.md | 1 + .../ceph/csi/example/cephfs/storageclass.yaml | 37 ++ .../ceph/csi/example/rbd/snapshotclass.yaml | 11 + .../ceph/csi/example/rbd/storageclass.yaml | 37 ++ .../csi/rbac/cephfs/csi-nodeplugin-rbac.yaml | 40 ++ .../csi/rbac/cephfs/csi-provisioner-rbac.yaml | 43 ++ .../ceph/csi/rbac/rbd/csi-attacher-rbac.yaml | 38 ++ .../csi/rbac/rbd/csi-nodeplugin-rbac.yaml | 41 ++ .../csi/rbac/rbd/csi-provisioner-rbac.yaml | 59 ++ .../cephfs/csi-cephfsplugin-provisioner.yaml | 78 +++ .../csi/template/cephfs/csi-cephfsplugin.yaml | 107 ++++ .../template/rbd/csi-rbdplugin-attacher.yaml | 35 ++ .../rbd/csi-rbdplugin-provisioner.yaml | 101 ++++ .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 116 ++++ .../kubernetes/ceph/operator-with-csi.yaml | 538 ++++++++++++++++++ cmd/rook/ceph/operator.go | 20 + .../csi/csi-cephfsplugin-provisioner.yaml | 1 + pkg/operator/ceph/csi/csi-cephfsplugin.yaml | 1 + .../ceph/csi/csi-rbdplugin-attacher.yaml | 1 + .../ceph/csi/csi-rbdplugin-provisioner.yaml | 1 + pkg/operator/ceph/csi/csi-rbdplugin.yaml | 1 + pkg/operator/ceph/csi/spec.go | 198 +++++++ pkg/operator/ceph/csi/spec_test.go | 46 ++ pkg/operator/ceph/csi/util.go | 67 +++ pkg/operator/ceph/csi/util_test.go | 108 ++++ pkg/operator/ceph/operator.go | 29 +- pkg/operator/k8sutil/apps.go | 82 +++ .../framework/installer/ceph_csi_templates.go | 467 +++++++++++++++ tests/framework/installer/ceph_installer.go | 15 + tests/framework/installer/ceph_manifests.go | 290 +++++++++- tests/integration/ceph_csi_test.go | 195 +++++++ tests/integration/smoke_test.go | 4 + 33 files changed, 2882 insertions(+), 118 deletions(-) create mode 100644 cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-nodeplugin-rbac.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-attacher-rbac.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml create mode 100644 cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml create mode 100755 cluster/examples/kubernetes/ceph/operator-with-csi.yaml create mode 120000 pkg/operator/ceph/csi/csi-cephfsplugin-provisioner.yaml create mode 120000 pkg/operator/ceph/csi/csi-cephfsplugin.yaml create mode 120000 pkg/operator/ceph/csi/csi-rbdplugin-attacher.yaml create mode 120000 pkg/operator/ceph/csi/csi-rbdplugin-provisioner.yaml create mode 120000 pkg/operator/ceph/csi/csi-rbdplugin.yaml create mode 100644 pkg/operator/ceph/csi/spec.go create mode 100644 pkg/operator/ceph/csi/spec_test.go create mode 100644 pkg/operator/ceph/csi/util.go create mode 100644 pkg/operator/ceph/csi/util_test.go create mode 100644 pkg/operator/k8sutil/apps.go create mode 100644 tests/framework/installer/ceph_csi_templates.go create mode 100644 tests/integration/ceph_csi_test.go diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/ceph-csi-drivers.md index f6440539d..105a68244 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/ceph-csi-drivers.md @@ -4,9 +4,9 @@ weight: 3200 indent: true --- -# Running ceph CSI drivers with Rook +# Running Ceph CSI drivers with Rook -Here is a guide on how to implement ceph-csi drivers on a Kubernetes cluster with Rook. The way to implement it should change relatively soon as there is an effort to manage deployments of the ceph-csi plugin within Rook directly ([#1385](https://github.com/rook/rook/issues/1385), [#2059](https://github.com/rook/rook/pull/2059)). +Here is a guide on how to use Rook to deploy ceph-csi drivers on a Kubernetes cluster. - [Enable CSI drivers](#enable-csi-drivers) - [Test the CSI driver](#test-the-csi-driver) @@ -15,134 +15,112 @@ Here is a guide on how to implement ceph-csi drivers on a Kubernetes cluster wit ## Prerequisites -1. A Kubernetes v1.12+ cluster with at least one node +1. a Kubernetes v1.13+ is needed in order to support CSI Spec 1.0. 2. `--allow-privileged` flag set to true in [kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) and your [API server](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) -3. An up and running Rook instance (see [Rook - ceph quickstart guide](https://github.com/rook/rook/blob/master/Documentation/ceph-quickstart.md)) +3. An up and running Rook instance (see [Rook - Ceph quickstart guide](https://github.com/rook/rook/blob/master/Documentation/ceph-quickstart.md)) -## Enable CSI drivers +## CSI Drivers Enablement -As described by the ceph-csi [Cephfs](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-cephfs.md) and [RBD](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-rbd.md) plugins deployment guides, the plugins yaml manifests are located in ceph-csi repository in the [deploy folder](https://github.com/ceph/ceph-csi/tree/master/deploy), and should be deployed as follow: +### Create RBAC used by CSI drivers in the same namespace as Rook Ceph Operator -### Deploy RBACs for node plugins, csi-attacher and csi-provisioner (common to Cephfs and RBD): -``` -kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-attacher-rbac.yaml -kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml -kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml -``` -Those manifests deploy service accounts, cluster roles and cluster role bindings. -### Deploy csi-attacher and csi-provisioner containers: -Deploys stateful sets for external-attacher and external-provisioner sidecar containers. -##### Cephfs - -``` -kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml -kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml -``` -##### RBD -``` -kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml -kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml -``` -### Deploy the CSI driver: -Deploys a daemon set with two containers: CSI driver-registrar and the driver. -##### Cephfs -``` -kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml -``` -##### RBD -``` -kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin.yaml +```console +kubectl create namespace rook-ceph-system +# create rbac. Since rook operator is not permitted to create rbac rules, these rules have to be created outside of operator +kubectl apply -f cluster/examples/kubernetes/ceph/csi/rbac/rbd/ +kubectl apply -f cluster/examples/kubernetes/ceph/csi/rbac/cephfs/ ``` -### Verify the plugin, its attacher and provisioner are running: -You should see an output similar to this when you run `kubectl get all`. +### Create CSI driver deployment templates and persist them in configmaps + +```console +kubectl create configmap csi-cephfs-config -n rook-ceph-system --from-file=cluster/examples/kubernetes/ceph/csi/template/cephfs + +kubectl create configmap csi-rbd-config -n rook-ceph-system --from-file=cluster/examples/kubernetes/ceph/csi/template/rbd ``` -# kubectl get all -NAMESPACE NAME READY STATUS RESTARTS AGE -default pod/csi-rbdplugin-975c4 2/2 Running 0 132m -default pod/csi-rbdplugin-attacher-0 1/1 Running 0 132m -default pod/csi-rbdplugin-provisioner-0 1/1 Running 0 132m +### Start Rook Ceph Operator -NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -default service/csi-rbdplugin-attacher ClusterIP 10.107.65.227 12345/TCP 132m -default service/csi-rbdplugin-provisioner ClusterIP 10.108.120.159 12345/TCP 132m +```console +kubectl apply -f cluster/examples/kubernetes/ceph/operator-with-csi.yaml +``` -NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE -default daemonset.apps/csi-rbdplugin 1 1 1 1 1 132m +### Verify CSI drivers and Operator are up and running -NAMESPACE NAME DESIRED CURRENT AGE -default statefulset.apps/csi-rbdplugin-attacher 1 1 132m -default statefulset.apps/csi-rbdplugin-provisioner 1 1 132m +```bash +# kubectl get all -n rook-ceph-system +NAME READY STATUS RESTARTS AGE +pod/csi-cephfsplugin-h5spd 2/2 Running 0 1d +pod/csi-cephfsplugin-provisioner-0 2/2 Running 0 1d +pod/csi-rbdplugin-4l6zg 2/2 Running 2 1d +pod/csi-rbdplugin-attacher-0 1/1 Running 0 1d +pod/csi-rbdplugin-provisioner-0 2/2 Running 2 1d +pod/rook-ceph-agent-zlm84 1/1 Running 0 1d +pod/rook-ceph-operator-c84954957-jdzk6 1/1 Running 0 1d +pod/rook-discover-66hjp 1/1 Running 0 1d +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/csi-cephfsplugin-provisioner ClusterIP 10.0.0.107 1234/TCP 1d +service/csi-rbdplugin-attacher ClusterIP 10.0.0.109 1234/TCP 1d +service/csi-rbdplugin-provisioner ClusterIP 10.0.0.56 1234/TCP 1d + +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE +daemonset.apps/csi-cephfsplugin 1 1 1 1 1 1d +daemonset.apps/csi-rbdplugin 1 1 1 1 1 1d +daemonset.apps/rook-ceph-agent 1 1 1 1 1 1d +daemonset.apps/rook-discover 1 1 1 1 1 1d + +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +deployment.apps/rook-ceph-operator 1 1 1 1 1d + +NAME DESIRED CURRENT READY AGE +replicaset.apps/rook-ceph-operator-c84954957 1 1 1 1d + +NAME DESIRED CURRENT AGE +statefulset.apps/csi-cephfsplugin-provisioner 1 1 1d +statefulset.apps/csi-rbdplugin-attacher 1 1 1d +statefulset.apps/csi-rbdplugin-provisioner 1 1 1d ``` ## Test the CSI driver -Once the plugin is successfully deployed, test it by running the following example +Once the plugin is successfully deployed, test it by running the following example. -This example is based on the ceph-csi [examples](https://github.com/ceph/ceph-csi/tree/master/examples) directory. It will describe how to test the RBD csi-driver, but it is very similar to run the cephfs one. +### Create the StorageClass -### Create the StorageClass: -This storageclass expect a pool named `rbd` in your ceph cluster. You can create this pool using [rook pool CRD](https://github.com/rook/rook/blob/master/Documentation/ceph-pool-crd.md) -##### storageclass.yaml +This [storageclass](../cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml) expect a pool named `rbd` in your Ceph cluster. You can create this pool using [rook pool CRD](https://github.com/rook/rook/blob/master/Documentation/ceph-pool-crd.md). + +Please update `monitors` to reflect the Ceph mointors. + + +### Create the Secret + +Create a Secret that matches that specified in the [storageclass](../cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml). + +Find a Ceph mon pod (in the following example, the pod is `rook-ceph-mon-a-6c4f9f6b6-rzp6r`) and create a Ceph user for that pool called `kubernetes`: +```bash +kubectl exec -ti -n rook-ceph rook-ceph-mon-a-6c4f9f6b6-rzp6r -- bash -c "ceph -c /var/lib/rook/rook-ceph/rook-ceph.config auth get-or-create-key client.kub2 mon \"allow profile rbd\" osd \"profile rbd pool=rbd\"" ``` -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-rbd -provisioner: csi-rbdplugin -parameters: - # Comma separated list of Ceph monitors - # if using FQDN, make sure csi plugin's dns policy is appropriate. - monitors: mon1:port,mon2:port,... - # if "monitors" parameter is not set, driver to get monitors from same - # secret as admin/user credentials. "monValueFromSecret" provides the - # key in the secret whose value is the mons - #monValueFromSecret: "monitors" +Then create a Secret using admin and `kubernetes` keyrings: - - # Ceph pool into which the RBD image shall be created - pool: rbd - - # RBD image format. Defaults to "2". - imageFormat: "2" - - # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. - imageFeatures: layering - - # The secrets have to contain Ceph admin credentials. - csiProvisionerSecretName: csi-rbd-secret - csiProvisionerSecretNamespace: default - csiNodePublishSecretName: csi-rbd-secret - csiNodePublishSecretNamespace: default - - # Ceph users for operating RBD - adminid: admin - userid: kubernetes - # uncomment the following to use rbd-nbd as mounter on supported nodes - #mounter: rbd-nbd -reclaimPolicy: Delete -``` -### Create the Secret: -##### secret.yaml ``` apiVersion: v1 kind: Secret metadata: name: csi-rbd-secret - namespace: default + namespace: default data: - # Key value corresponds to a user name defined in ceph cluster + # Key value corresponds to a user name defined in Ceph cluster admin: BASE64-ENCODED-PASSWORD - # Key value corresponds to a user name defined in ceph cluster + # Key value corresponds to a user name defined in Ceph cluster kubernetes: BASE64-ENCODED-PASSWORD # if monValueFromSecret is set to "monitors", uncomment the # following and set the mon there #monitors: BASE64-ENCODED-Comma-Delimited-Mons ``` -Here, you need your ceph admin/user password encoded in base64. Run `ceph auth ls` in one of your ceph pod, encode the key of your admin/user and replace `BASE64-ENCODED-PASSWORD` by your encoded key. + +Here, you need your Ceph admin/user password encoded in base64. Run `ceph auth ls` in one of your Ceph pod, encode the key of your admin/user and replace `BASE64-ENCODED-PASSWORD` by your encoded key. + ### Create the PersistentVolumeClaim: ##### pvc.yaml ``` @@ -188,7 +166,7 @@ spec: readOnly: false ``` -When running `rbd list block --pool [yourpool]` in one of your ceph pod you should see the created PVC: +When running `rbd list block --pool [yourpool]` in one of your Ceph pod you should see the created PVC: ``` # rbd list block --pool rbd pvc-c20495c0d5de11e8 @@ -220,7 +198,7 @@ You must download this file and modify it to match your Ceph cluster. ``` # wget https://raw.githubusercontent.com/ceph/ceph-csi/master/examples/rbd/snapshotclass.yaml ``` -The `csiSnapshotterSecretName` parameter should reference the name of the secret created for the ceph-csi plugin you deployed. The monitors are a comma separated list of your ceph monitors, same as in the StorageClass of the plugin you chosen. When this is done, run: +The `csiSnapshotterSecretName` parameter should reference the name of the secret created for the ceph-csi plugin you deployed. The monitors are a comma separated list of your Ceph monitors, same as in the StorageClass of the plugin you chosen. When this is done, run: ``` # kubectl create -f snapshotclass.yaml ``` @@ -238,7 +216,7 @@ csi-rbdplugin-snapclass 4s NAME AGE rbd-pvc-snapshot 6s ``` -In one of your ceph pod, run `rbd snap ls [name-of-your-pvc]`. +In one of your Ceph pod, run `rbd snap ls [name-of-your-pvc]`. The output should be similar to this: ``` # rbd snap ls pvc-c20495c0d5de11e8 @@ -256,20 +234,8 @@ To clean your cluster of the resources created by this example, run the followin # kubectl delete -f pvc.yaml # kubectl delete -f secret.yaml # kubectl delete -f storageclass.yaml - -# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin.yaml -# Or https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml if you deployed cephfs plugin - -# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml -# Or https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml - -# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml -# Or https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml - -# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml -# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml -# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-attacher-rbac.yaml ``` + If you tested snapshots too: ``` # kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/examples/rbd/snapshot.yaml diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index f934039de..7e826e1a7 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -17,6 +17,7 @@ - Recursive chown for mounts can now be toggled with the [ROOK_ENABLE_FSGROUP](https://github.com/rook/rook/issues/2254) environment variable. - Added the dashboard `port` configuration setting. - Added the dashboard `ssl` configuration setting. +- Added Ceph CSI driver deployments on Kubernetes 1.13 and above. ## Breaking Changes diff --git a/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml b/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml new file mode 100644 index 000000000..c23814322 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/example/cephfs/storageclass.yaml @@ -0,0 +1,37 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-cephfs +provisioner: csi-cephfsplugin +parameters: + # Comma separated list of Ceph monitors + # if using FQDN, make sure csi plugin's dns policy is appropriate. + monitors: mon1:port,mon2:port,... + + # For provisionVolume: "true": + # A new volume will be created along with a new Ceph user. + # Requires admin credentials (adminID, adminKey). + # For provisionVolume: "false": + # It is assumed the volume already exists and the user is expected + # to provide path to that volume (rootPath) and user credentials (userID, userKey). + provisionVolume: "true" + + # Ceph pool into which the volume shall be created + # Required for provisionVolume: "true" + pool: cephfs_data + + # Root path of an existing CephFS volume + # Required for provisionVolume: "false" + # rootPath: /absolute/path + + # The secrets have to contain user and/or Ceph admin credentials. + csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret + csi.storage.k8s.io/provisioner-secret-namespace: default + csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret + csi.storage.k8s.io/node-stage-secret-namespace: default + + # (optional) The driver can use either ceph-fuse (fuse) or ceph kernel client (kernel) + # If omitted, default volume mounter will be used - this is determined by probing for ceph-fuse + # or by setting the default mounter explicitly via --volumemounter command-line argument. + # mounter: kernel +reclaimPolicy: Delete diff --git a/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml b/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml new file mode 100644 index 000000000..778f2b084 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1alpha1 +kind: VolumeSnapshotClass +metadata: + name: csi-rbdplugin-snapclass +snapshotter: csi-rbdplugin +parameters: + pool: rbd + monitors: mon1:port,mon2:port,... + csi.storage.k8s.io/snapshotter-secret-name: csi-rbd-secret + csi.storage.k8s.io/snapshotter-secret-namespace: default diff --git a/cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml b/cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml new file mode 100644 index 000000000..d1a568ce7 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml @@ -0,0 +1,37 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-rbd +provisioner: csi-rbdplugin +parameters: + # Comma separated list of Ceph monitors + # if using FQDN, make sure csi plugin's dns policy is appropriate. + monitors: mon1:port,mon2:port,... + + # if "monitors" parameter is not set, driver to get monitors from same + # secret as admin/user credentials. "monValueFromSecret" provides the + # key in the secret whose value is the mons + #monValueFromSecret: "monitors" + + + # Ceph pool into which the RBD image shall be created + pool: rbd + + # RBD image format. Defaults to "2". + imageFormat: "2" + + # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. + imageFeatures: layering + + # The secrets have to contain Ceph admin credentials. + csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret + csi.storage.k8s.io/provisioner-secret-namespace: default + csi.storage.k8s.io/node-publish-secret-name: csi-rbd-secret + csi.storage.k8s.io/node-publish-secret-namespace: default + + # Ceph users for operating RBD + adminid: admin + userid: kubernetes + # uncomment the following to use rbd-nbd as mounter on supported nodes + #mounter: rbd-nbd +reclaimPolicy: Delete diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-nodeplugin-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-nodeplugin-rbac.yaml new file mode 100644 index 000000000..03aee954a --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-nodeplugin-rbac.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-plugin-sa + namespace: rook-ceph-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: rook-ceph-system +roleRef: + kind: ClusterRole + name: cephfs-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml new file mode 100644 index 000000000..182e65019 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/rbac/cephfs/csi-provisioner-rbac.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-provisioner-sa + namespace: rook-ceph-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: rook-ceph-system +roleRef: + kind: ClusterRole + name: cephfs-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-attacher-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-attacher-rbac.yaml new file mode 100644 index 000000000..10178983f --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-attacher-rbac.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-attacher-sa + namespace: rook-ceph-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-attacher-runner +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-attacher-role +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-attacher-sa + namespace: rook-ceph-system +roleRef: + kind: ClusterRole + name: rbd-external-attacher-runner + apiGroup: rbac.authorization.k8s.io diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml new file mode 100644 index 000000000..a47f573d9 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-nodeplugin-rbac.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph-system +roleRef: + kind: ClusterRole + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io diff --git a/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml new file mode 100644 index 000000000..f416b0d6d --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/rbac/rbd/csi-provisioner-rbac.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "create", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph-system +roleRef: + kind: ClusterRole + name: rbd-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml new file mode 100644 index 000000000..e37bd6d9c --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml @@ -0,0 +1,78 @@ +kind: StatefulSet +apiVersion: apps/v1beta2 +metadata: + name: csi-cephfsplugin-provisioner + namespace: {{ .Namespace }} +spec: + serviceName: "csi-cephfsplugin-provisioner" + replicas: 1 + selector: + matchLabels: + app: csi-cephfsplugin-provisioner + template: + metadata: + labels: + app: csi-cephfsplugin-provisioner + spec: + serviceAccount: rook-csi-cephfs-provisioner-sa + containers: + - name: csi-provisioner + image: {{ .ProvisionerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + image: {{ .CephFSPluginImage }} + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=csi-cephfsplugin" + - "--metadatastorage=k8s_configmap" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-cephfsplugin + type: DirectoryOrCreate + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml new file mode 100644 index 000000000..9b121933a --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -0,0 +1,107 @@ +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: csi-cephfsplugin + namespace: {{ .Namespace }} +spec: + selector: + matchLabels: + app: csi-cephfsplugin + template: + metadata: + labels: + app: csi-cephfsplugin + spec: + serviceAccount: rook-csi-cephfs-plugin-sa + hostNetwork: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: {{ .RegistrarImage }} + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-cephfsplugin /registration/csi-cephfsplugin-reg.sock"] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: {{ .CephFSPluginImage }} + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=csi-cephfsplugin" + - "--metadatastorage=k8s_configmap" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + - name: csi-plugins-dir + mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi + mountPropagation: "Bidirectional" + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-cephfsplugin/ + type: DirectoryOrCreate + - name: csi-plugins-dir + hostPath: + path: /var/lib/kubelet/plugins/kubernetes.io/csi + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml new file mode 100644 index 000000000..b4de4dfde --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml @@ -0,0 +1,35 @@ +kind: StatefulSet +apiVersion: apps/v1beta2 +metadata: + name: csi-rbdplugin-attacher + namespace: {{ .Namespace }} +spec: + serviceName: csi-rbdplugin-attacher + replicas: 1 + selector: + matchLabels: + app: csi-rbdplugin-attacher + template: + metadata: + labels: + app: csi-rbdplugin-attacher + spec: + serviceAccount: rook-csi-rbd-attacher-sa + containers: + - name: csi-rbdplugin-attacher + image: {{ .AttacherImage }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-rbdplugin + type: DirectoryOrCreate diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml new file mode 100644 index 000000000..ac167e750 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml @@ -0,0 +1,101 @@ +kind: StatefulSet +apiVersion: apps/v1beta2 +metadata: + name: csi-rbdplugin-provisioner + namespace: {{ .Namespace }} +spec: + serviceName: "csi-rbdplugin-provisioner" + replicas: 1 + selector: + matchLabels: + app: csi-rbdplugin-provisioner + template: + metadata: + labels: + app: csi-rbdplugin-provisioner + spec: + serviceAccount: rook-csi-rbd-provisioner-sa + containers: + - name: csi-provisioner + image: {{ .ProvisionerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + - name: csi-snapshotter + image: {{ .SnapshotterImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--connection-timeout=15s" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + imagePullPolicy: Always + securityContext: + privileged: true + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + image: {{ .RBDPluginImage }} + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=csi-rbdplugin" + - "--containerized=true" + - "--metadatastorage=k8s_configmap" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + volumes: + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-rbdplugin + type: DirectoryOrCreate diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml new file mode 100644 index 000000000..eaecaca3d --- /dev/null +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -0,0 +1,116 @@ +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: csi-rbdplugin + namespace: {{ .Namespace }} +spec: + selector: + matchLabels: + app: csi-rbdplugin + template: + metadata: + labels: + app: csi-rbdplugin + spec: + serviceAccount: rook-csi-rbd-plugin-sa + hostNetwork: true + hostPID: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: {{ .RegistrarImage }} + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: {{ .RBDPluginImage }} + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=csi-rbdplugin" + - "--containerized=true" + - "--metadatastorage=k8s_configmap" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: plugin-mount-dir + mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/ + mountPropagation: "Bidirectional" + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-rbdplugin + type: DirectoryOrCreate + - name: plugin-mount-dir + hostPath: + path: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules diff --git a/cluster/examples/kubernetes/ceph/operator-with-csi.yaml b/cluster/examples/kubernetes/ceph/operator-with-csi.yaml new file mode 100755 index 000000000..a56667913 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/operator-with-csi.yaml @@ -0,0 +1,538 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: rook-ceph-system +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephclusters.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephCluster + listKind: CephClusterList + plural: cephclusters + singular: cephcluster + scope: Namespaced + version: v1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + cephVersion: + properties: + allowUnsupported: + type: boolean + image: + type: string + name: + pattern: ^(luminous|mimic|nautilus)$ + type: string + dashboard: + properties: + enabled: + type: boolean + urlPrefix: + type: string + port: + type: integer + dataDirHostPath: + pattern: ^/(\S+) + type: string + mon: + properties: + allowMultiplePerNode: + type: boolean + count: + maximum: 9 + minimum: 1 + type: integer + required: + - count + network: + properties: + hostNetwork: + type: boolean + storage: + properties: + nodes: + items: {} + type: array + useAllDevices: {} + useAllNodes: + type: boolean + required: + - mon + additionalPrinterColumns: + - name: DataDirHostPath + type: string + description: Directory used on the K8s nodes + JSONPath: .spec.dataDirHostPath + - name: MonCount + type: string + description: Number of MONs + JSONPath: .spec.mon.count + - name: Age + type: date + JSONPath: .metadata.creationTimestamp + - name: State + type: string + description: Current State + JSONPath: .status.state +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephfilesystems.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephFilesystem + listKind: CephFilesystemList + plural: cephfilesystems + singular: cephfilesystem + scope: Namespaced + version: v1 + additionalPrinterColumns: + - name: MdsCount + type: string + description: Number of MDSs + JSONPath: .spec.metadataServer.activeCount + - name: Age + type: date + JSONPath: .metadata.creationTimestamp +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephnfses.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephNFS + listKind: CephNFSList + plural: cephnfses + singular: cephnfs + shortNames: + - nfs + scope: Namespaced + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephobjectstores.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectStore + listKind: CephObjectStoreList + plural: cephobjectstores + singular: cephobjectstore + scope: Namespaced + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephobjectstoreusers.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectStoreUser + listKind: CephObjectStoreUserList + plural: cephobjectstoreusers + singular: cephobjectstoreuser + scope: Namespaced + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephblockpools.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBlockPool + listKind: CephBlockPoolList + plural: cephblockpools + singular: cephblockpool + scope: Namespaced + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: volumes.rook.io +spec: + group: rook.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + singular: volume + shortNames: + - rv + scope: Namespaced + version: v1alpha2 +--- +# The cluster role for managing all the cluster-specific resources in a namespace +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-cluster-mgmt + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + - secrets + - pods + - pods/log + - services + - configmaps + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - extensions + resources: + - deployments + - daemonsets + - replicasets + verbs: + - get + - list + - watch + - create + - update + - delete +--- +# The role for the operator to manage resources in the system namespace +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: rook-ceph-system + namespace: rook-ceph-system + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - extensions + resources: + - daemonsets + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - apps + resources: + - daemonsets + - statefulsets + verbs: + - get + - list + - watch + - create + - update + - delete + +--- +# The cluster role for managing the Rook CRDs +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + # Pod access is needed for fencing + - pods + # Node access is needed for determining nodes where mons should run + - nodes + - nodes/proxy + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + # PVs and PVCs are managed by the Rook provisioner + - persistentvolumes + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - ceph.rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" +--- +# Aspects of ceph-mgr that require cluster-wide access +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-mgr-cluster + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + - configmaps + - nodes + - nodes/proxy + verbs: + - get + - list + - watch +--- +# The rook system service account used by the operator, agent, and discovery pods +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-system + namespace: rook-ceph-system + labels: + operator: rook + storage-backend: ceph +--- +# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-system + namespace: rook-ceph-system + labels: + operator: rook + storage-backend: ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-system +subjects: +- kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph-system +--- +# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-global + namespace: rook-ceph-system + labels: + operator: rook + storage-backend: ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-global +subjects: +- kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph-system +--- +# The deployment for the rook operator +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: rook-ceph-operator + namespace: rook-ceph-system + labels: + operator: rook + storage-backend: ceph +spec: + replicas: 1 + template: + metadata: + labels: + app: rook-ceph-operator + spec: + serviceAccountName: rook-ceph-system + containers: + - name: rook-ceph-operator + image: rook/ceph:master + args: ["ceph", "operator"] + volumeMounts: + - mountPath: /var/lib/rook + name: rook-config + - mountPath: /etc/ceph + name: default-config-dir + - mountPath: /etc/ceph-csi/rbd + name: csi-rbd-config + - mountPath: /etc/ceph-csi/cephfs + name: csi-cephfs-config + env: + # To disable RBAC, uncomment the following: + # - name: RBAC_ENABLED + # value: "false" + # Rook Agent toleration. Will tolerate all taints with all keys. + # Choose between NoSchedule, PreferNoSchedule and NoExecute: + # - name: AGENT_TOLERATION + # value: "NoSchedule" + # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate + # - name: AGENT_TOLERATION_KEY + # value: "" + # (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`. + # `Any` uses Ceph admin credentials by default/fallback. + # For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and + # set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name. + # to the namespace in which the `mountSecret` Kubernetes secret namespace. + # - name: AGENT_MOUNT_SECURITY_MODE + # value: "Any" + # Set the path where the Rook agent can find the flex volumes + # - name: FLEXVOLUME_DIR_PATH + # value: "" + # Set the path where kernel modules can be found + # - name: LIB_MODULES_DIR_PATH + # value: "" + # Mount any extra directories into the agent container + # - name: AGENT_MOUNTS + # value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2" + # Rook Discover toleration. Will tolerate all taints with all keys. + # Choose between NoSchedule, PreferNoSchedule and NoExecute: + # - name: DISCOVER_TOLERATION + # value: "NoSchedule" + # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate + # - name: DISCOVER_TOLERATION_KEY + # value: "" + # Allow rook to create multiple file systems. Note: This is considered + # an experimental feature in Ceph as described at + # http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster + # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027 + - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS + value: "false" + # The logging level for the operator: INFO | DEBUG + - name: ROOK_LOG_LEVEL + value: "INFO" + # The interval to check if every mon is in the quorum. + - name: ROOK_MON_HEALTHCHECK_INTERVAL + value: "45s" + # The duration to wait before trying to failover or remove/replace the + # current mon with a new mon (useful for compensating flapping network). + - name: ROOK_MON_OUT_TIMEOUT + value: "300s" + # The duration between discovering devices in the rook-discover daemonset. + - name: ROOK_DISCOVER_DEVICES_INTERVAL + value: "60m" + # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods. + # This is necessary to workaround the anyuid issues when running on OpenShift. + # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 + - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED + value: "false" + # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins). + # Disable it here if you have similiar issues. + # For more details see https://github.com/rook/rook/issues/2417 + - name: ROOK_ENABLE_SELINUX_RELABELING + value: "true" + # CSI enablement + - name: ROOK_CSI_ENABLE_CEPHFS + value: "true" + - name: ROOK_CSI_CEPHFS_IMAGE + value: "quay.io/cephcsi/cephfsplugin:v1.0.0" + - name: ROOK_CSI_ENABLE_RBD + value: "true" + - name: ROOK_CSI_RBD_IMAGE + value: "quay.io/cephcsi/rbdplugin:v1.0.0" + - name: ROOK_CSI_REGISTRAR_IMAGE + value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2" + - name: ROOK_CSI_PROVISIONER_IMAGE + value: "quay.io/k8scsi/csi-provisioner:v1.0.1" + - name: ROOK_CSI_SNAPSHOTTER_IMAGE + value: "quay.io/k8scsi/csi-snapshotter:v1.0.1" + - name: ROOK_CSI_ATTACHER_IMAGE + value: "quay.io/k8scsi/csi-attacher:v1.0.1" + # The name of the node to pass with the downward API + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # The pod name to pass with the downward API + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # The pod namespace to pass with the downward API + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: rook-config + emptyDir: {} + - name: default-config-dir + emptyDir: {} + - name: csi-rbd-config + configMap: + name: csi-rbd-config + - name: csi-cephfs-config + configMap: + name: csi-cephfs-config diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index 63c41dfc4..f55a3936b 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -24,6 +24,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment" "github.com/rook/rook/pkg/operator/ceph" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" + "github.com/rook/rook/pkg/operator/ceph/csi" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" @@ -42,6 +43,25 @@ https://github.com/rook/rook`, func init() { operatorCmd.Flags().DurationVar(&mon.HealthCheckInterval, "mon-healthcheck-interval", mon.HealthCheckInterval, "mon health check interval (duration)") operatorCmd.Flags().DurationVar(&mon.MonOutTimeout, "mon-out-timeout", mon.MonOutTimeout, "mon out timeout (duration)") + + operatorCmd.Flags().BoolVar(&csi.EnableRBD, "csi-enable-rbd", false, "whether enable ceph-csi rbd driver") + operatorCmd.Flags().BoolVar(&csi.EnableCephFS, "csi-enable-cephfs", false, "whether enable ceph-csi cephfs driver") + // csi images + operatorCmd.Flags().StringVar(&csi.CSIParam.RBDPluginImage, "csi-rbd-image", csi.DefaultRBDPluginImage, "ceph-csi rbd plugin image") + operatorCmd.Flags().StringVar(&csi.CSIParam.CephFSPluginImage, "csi-cephfs-image", csi.DefaultCephFSPluginImage, "ceph-csi cephfs plugin image") + operatorCmd.Flags().StringVar(&csi.CSIParam.RegistrarImage, "csi-registrar-image", csi.DefaultRegistrarImage, "csi registrar image") + operatorCmd.Flags().StringVar(&csi.CSIParam.ProvisionerImage, "csi-provisioner-image", csi.DefaultProvisionerImage, "csi provisioner image") + operatorCmd.Flags().StringVar(&csi.CSIParam.AttacherImage, "csi-attacher-image", csi.DefaultAttacherImage, "csi attacher image") + operatorCmd.Flags().StringVar(&csi.CSIParam.SnapshotterImage, "csi-snapshotter-image", csi.DefaultSnapshotterImage, "csi snapshotter image") + + // csi deployment templates + operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template") + operatorCmd.Flags().StringVar(&csi.RBDProvisionerTemplatePath, "csi-rbd-provisioner-template-path", csi.DefaultRBDProvisionerTemplatePath, "path to ceph-csi rbd provisioner template") + operatorCmd.Flags().StringVar(&csi.RBDAttacherTemplatePath, "csi-rbd-attacher-template-path", csi.DefaultRBDAttacherTemplatePath, "path to ceph-csi rbd attacher template") + + operatorCmd.Flags().StringVar(&csi.CephFSPluginTemplatePath, "csi-cephfs-plugin-template-path", csi.DefaultCephFSPluginTemplatePath, "path to ceph-csi cephfs plugin template") + operatorCmd.Flags().StringVar(&csi.CephFSProvisionerTemplatePath, "csi-cephfs-provisioner-template-path", csi.DefaultCephFSProvisionerTemplatePath, "path to ceph-csi cephfs provisioner template") + flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix) flags.SetLoggingFlags(operatorCmd.Flags()) operatorCmd.RunE = startOperator diff --git a/pkg/operator/ceph/csi/csi-cephfsplugin-provisioner.yaml b/pkg/operator/ceph/csi/csi-cephfsplugin-provisioner.yaml new file mode 120000 index 000000000..761be2cf1 --- /dev/null +++ b/pkg/operator/ceph/csi/csi-cephfsplugin-provisioner.yaml @@ -0,0 +1 @@ +../../../../cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml \ No newline at end of file diff --git a/pkg/operator/ceph/csi/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/csi-cephfsplugin.yaml new file mode 120000 index 000000000..7e0d66e54 --- /dev/null +++ b/pkg/operator/ceph/csi/csi-cephfsplugin.yaml @@ -0,0 +1 @@ +../../../../cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml \ No newline at end of file diff --git a/pkg/operator/ceph/csi/csi-rbdplugin-attacher.yaml b/pkg/operator/ceph/csi/csi-rbdplugin-attacher.yaml new file mode 120000 index 000000000..2739e0de4 --- /dev/null +++ b/pkg/operator/ceph/csi/csi-rbdplugin-attacher.yaml @@ -0,0 +1 @@ +../../../../cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml \ No newline at end of file diff --git a/pkg/operator/ceph/csi/csi-rbdplugin-provisioner.yaml b/pkg/operator/ceph/csi/csi-rbdplugin-provisioner.yaml new file mode 120000 index 000000000..af612bbf8 --- /dev/null +++ b/pkg/operator/ceph/csi/csi-rbdplugin-provisioner.yaml @@ -0,0 +1 @@ +../../../../cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml \ No newline at end of file diff --git a/pkg/operator/ceph/csi/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/csi-rbdplugin.yaml new file mode 120000 index 000000000..d4f95de87 --- /dev/null +++ b/pkg/operator/ceph/csi/csi-rbdplugin.yaml @@ -0,0 +1 @@ +../../../../cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml \ No newline at end of file diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go new file mode 100644 index 000000000..c108d09a0 --- /dev/null +++ b/pkg/operator/ceph/csi/spec.go @@ -0,0 +1,198 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "fmt" + + "github.com/rook/rook/pkg/operator/k8sutil" + + apps "k8s.io/api/apps/v1beta2" + "k8s.io/client-go/kubernetes" +) + +type Param struct { + Namespace string + + RBDPluginImage string + CephFSPluginImage string + RegistrarImage string + ProvisionerImage string + AttacherImage string + SnapshotterImage string +} + +var ( + CSIParam Param + + EnableRBD = true + EnableCephFS = true + + // template paths + RBDPluginTemplatePath string + RBDProvisionerTemplatePath string + RBDAttacherTemplatePath string + + CephFSPluginTemplatePath string + CephFSProvisionerTemplatePath string +) + +const ( + KubeMinMajor = "1" + KubeMinMinor = "13" + + // image names + DefaultRBDPluginImage = "quay.io/cephcsi/rbdplugin:v1.0.0" + DefaultCephFSPluginImage = "quay.io/cephcsi/cephfsplugin:v1.0.0" + DefaultRegistrarImage = "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2" + DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.0.1" + DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v1.0.1" + DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.0.1" + + // template + DefaultRBDPluginTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml" + DefaultRBDProvisionerTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml" + DefaultRBDAttacherTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-attacher.yaml" + DefaultCephFSPluginTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml" + DefaultCephFSProvisionerTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner.yaml" + + ExitOnError = false // don't exit if CSI fails to deploy. Switch to true when flexdriver is disabled +) + +func CSIEnabled() bool { + if EnableRBD || EnableCephFS { + return true + } + return false +} + +func SetCSINamespace(namespace string) { + CSIParam.Namespace = namespace +} + +func ValidateCSIParam() error { + if EnableRBD { + if len(CSIParam.RBDPluginImage) == 0 { + return fmt.Errorf("missing csi rbd plugin image") + } + if len(CSIParam.RegistrarImage) == 0 { + return fmt.Errorf("missing csi registrar image") + } + if len(CSIParam.ProvisionerImage) == 0 { + return fmt.Errorf("missing csi provisioner image") + } + if len(CSIParam.AttacherImage) == 0 { + return fmt.Errorf("missing csi attacher image") + } + if len(RBDPluginTemplatePath) == 0 { + return fmt.Errorf("missing rbd plugin template path") + } + if len(RBDProvisionerTemplatePath) == 0 { + return fmt.Errorf("missing rbd provisioner template path") + } + if len(RBDAttacherTemplatePath) == 0 { + return fmt.Errorf("missing rbd attacher template path") + } + } + + if EnableCephFS { + if len(CSIParam.CephFSPluginImage) == 0 { + return fmt.Errorf("missing csi cephfs plugin image") + } + if len(CSIParam.RegistrarImage) == 0 { + return fmt.Errorf("missing csi registrar image") + } + if len(CSIParam.ProvisionerImage) == 0 { + return fmt.Errorf("missing csi provisioner image") + } + if len(CephFSPluginTemplatePath) == 0 { + return fmt.Errorf("missing cephfs plugin template path") + } + if len(CephFSProvisionerTemplatePath) == 0 { + return fmt.Errorf("missing ceph provisioner template path") + } + } + return nil +} + +func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error { + var ( + err error + rbdPlugin, cephfsPlugin *apps.DaemonSet + rbdProvisioner, rbdAttacher, cephfsProvisioner *apps.StatefulSet + ) + + if EnableRBD { + rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath) + if err != nil { + return fmt.Errorf("failed to load rbd plugin template: %v", err) + } + rbdProvisioner, err = templateToStatefulSet("rbd-provisioner", RBDProvisionerTemplatePath) + if err != nil { + return fmt.Errorf("failed to load rbd provisioner template: %v", err) + } + rbdAttacher, err = templateToStatefulSet("rbd-attacher", RBDAttacherTemplatePath) + if err != nil { + return fmt.Errorf("failed to load rbd attacher template: %v", err) + } + } + if EnableCephFS { + cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath) + if err != nil { + return fmt.Errorf("failed to load CephFS plugin template: %v", err) + } + cephfsProvisioner, err = templateToStatefulSet("cephfs-provisioner", CephFSProvisionerTemplatePath) + if err != nil { + return fmt.Errorf("failed to load CephFS provisioner template: %v", err) + } + } + + if rbdPlugin != nil { + err = k8sutil.CreateDaemonSet("csi rbd plugin", namespace, clientset, rbdPlugin) + if err != nil { + return fmt.Errorf("failed to start rbdplugin daemonset: %v\n%v", err, rbdPlugin) + } + } + if rbdProvisioner != nil { + _, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisioner) + if err != nil { + return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisioner) + } + + } + if rbdAttacher != nil { + _, err = k8sutil.CreateStatefulSet("csi rbd attacher", namespace, "csi-rbdplugin-attacher", clientset, rbdAttacher) + if err != nil { + return fmt.Errorf("failed to start rbd attacher statefulset: %v\n%v", err, rbdAttacher) + } + } + + if cephfsPlugin != nil { + err = k8sutil.CreateDaemonSet("csi cephfs plugin", namespace, clientset, cephfsPlugin) + if err != nil { + return fmt.Errorf("failed to start cephfs plugin daemonset: %v\n%v", err, cephfsPlugin) + } + } + if cephfsProvisioner != nil { + _, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisioner) + if err != nil { + return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisioner) + } + + } + return nil +} diff --git a/pkg/operator/ceph/csi/spec_test.go b/pkg/operator/ceph/csi/spec_test.go new file mode 100644 index 000000000..f7a71acba --- /dev/null +++ b/pkg/operator/ceph/csi/spec_test.go @@ -0,0 +1,46 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "testing" + + "github.com/rook/rook/pkg/operator/test" + + "github.com/stretchr/testify/assert" +) + +func TestStartCSI(t *testing.T) { + RBDPluginTemplatePath = "csi-rbdplugin.yaml" + RBDProvisionerTemplatePath = "csi-rbdplugin-provisioner.yaml" + RBDAttacherTemplatePath = "csi-rbdplugin-attacher.yaml" + + CephFSPluginTemplatePath = "csi-cephfsplugin.yaml" + CephFSProvisionerTemplatePath = "csi-cephfsplugin-provisioner.yaml" + + CSIParam = Param{ + RBDPluginImage: "image", + CephFSPluginImage: "image", + RegistrarImage: "image", + ProvisionerImage: "image", + AttacherImage: "image", + SnapshotterImage: "image", + } + clientset := test.New(3) + err := StartCSIDrivers("ns", clientset) + assert.Nil(t, err) +} diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go new file mode 100644 index 000000000..9694ac408 --- /dev/null +++ b/pkg/operator/ceph/csi/util.go @@ -0,0 +1,67 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "bytes" + "io/ioutil" + "text/template" + + "github.com/ghodss/yaml" + + apps "k8s.io/api/apps/v1beta2" +) + +func loadTemplate(name, templatePath string) (string, error) { + b, err := ioutil.ReadFile(templatePath) + if err != nil { + return "", err + } + data := string(b) + var writer bytes.Buffer + t := template.New(name) + err = template.Must(t.Parse(data)).Execute(&writer, CSIParam) + return writer.String(), err +} + +func templateToStatefulSet(name, templatePath string) (*apps.StatefulSet, error) { + var ss apps.StatefulSet + t, err := loadTemplate(name, templatePath) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal([]byte(t), &ss) + if err != nil { + return nil, err + } + return &ss, nil +} + +func templateToDaemonSet(name, templatePath string) (*apps.DaemonSet, error) { + var ds apps.DaemonSet + t, err := loadTemplate(name, templatePath) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal([]byte(t), &ds) + if err != nil { + return nil, err + } + return &ds, nil +} diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go new file mode 100644 index 000000000..76856f44a --- /dev/null +++ b/pkg/operator/ceph/csi/util_test.go @@ -0,0 +1,108 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +var ( + testDSTemplate = []byte(` +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: test-label + namespace: {{ .Namespace }} +spec: + selector: + matchLabels: + app: test-label + template: + metadata: + labels: + app: test-label + spec: + serviceAccount: test-sa + containers: + - name: registrar + image: {{ .RegistrarImage }} + - name: rbdplugin + image: {{ .RBDPluginImage }} + - name: cephfsplugin + image: {{ .CephFSPluginImage }} +`) + testSSTemplate = []byte(` +kind: StatefulSet +apiVersion: apps/v1beta2 +metadata: + name: test-label + namespace: {{ .Namespace }} +spec: + selector: + matchLabels: + app: test-label + template: + metadata: + labels: + app: test-label + spec: + serviceAccount: test-sa + containers: + - name: csi-rbdplugin-attacher + image: {{ .AttacherImage }} + - name: provisioner + image: {{ .ProvisionerImage }} + - name: rbdplugin + image: {{ .RBDPluginImage }} + - name: cephfsplugin + image: {{ .CephFSPluginImage }} +`) +) + +func TestDaemonSetTemplate(t *testing.T) { + tmp, err := ioutil.TempFile("", "yaml") + assert.Nil(t, err) + + defer os.Remove(tmp.Name()) + + _, err = tmp.Write(testDSTemplate) + assert.Nil(t, err) + err = tmp.Close() + assert.Nil(t, err) + + _, err = templateToDaemonSet("test-ds", tmp.Name()) + assert.Nil(t, err) +} + +func TestStatefulSetTemplate(t *testing.T) { + tmp, err := ioutil.TempFile("", "yaml") + assert.Nil(t, err) + + defer os.Remove(tmp.Name()) + + _, err = tmp.Write(testSSTemplate) + assert.Nil(t, err) + err = tmp.Close() + assert.Nil(t, err) + + _, err = templateToStatefulSet("test-ss", tmp.Name()) + assert.Nil(t, err) +} diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index b35d2ac3f..143e9bb39 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -31,6 +31,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment" "github.com/rook/rook/pkg/operator/ceph/agent" "github.com/rook/rook/pkg/operator/ceph/cluster" + "github.com/rook/rook/pkg/operator/ceph/csi" "github.com/rook/rook/pkg/operator/ceph/file" "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/pkg/operator/ceph/object/user" @@ -100,15 +101,35 @@ func (o *Operator) Run() error { return fmt.Errorf("Error starting device discovery daemonset: %v", err) } - signalChan := make(chan os.Signal, 1) - stopChan := make(chan struct{}) - signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) - serverVersion, err := o.context.Clientset.Discovery().ServerVersion() if err != nil { return fmt.Errorf("Error getting server version: %v", err) } + if serverVersion.Major >= csi.KubeMinMajor && serverVersion.Minor >= csi.KubeMinMinor && csi.CSIEnabled() { + logger.Infof("Ceph CSI driver is enabled, validate csi param") + if err = csi.ValidateCSIParam(); err != nil { + logger.Warningf("invalid csi params: %v", err) + if csi.ExitOnError { + return err + } + } else { + csi.SetCSINamespace(namespace) + if err = csi.StartCSIDrivers(namespace, o.context.Clientset); err != nil { + logger.Warningf("failed to start Ceph csi drivers: %v", err) + if csi.ExitOnError { + return err + } + } else { + logger.Infof("successfully started Ceph csi drivers") + } + } + } + + signalChan := make(chan os.Signal, 1) + stopChan := make(chan struct{}) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + // Run volume provisioner for each of the supported configurations for name, vendor := range provisionerConfigs { volumeProvisioner := provisioner.New(o.context, vendor) diff --git a/pkg/operator/k8sutil/apps.go b/pkg/operator/k8sutil/apps.go new file mode 100644 index 000000000..8bb6dfa38 --- /dev/null +++ b/pkg/operator/k8sutil/apps.go @@ -0,0 +1,82 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8sutil + +import ( + "fmt" + + apps "k8s.io/api/apps/v1beta2" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// make a headless svc for statefulset +func makeHeadlessSvc(name, namespace string, label map[string]string, clientset kubernetes.Interface) (*corev1.Service, error) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: label, + }, + Spec: corev1.ServiceSpec{ + Selector: label, + Ports: []corev1.ServicePort{{Name: "dummy", Port: 1234}}, + }, + } + + _, err := clientset.CoreV1().Services(namespace).Create(svc) + if err != nil && !k8serrors.IsAlreadyExists(err) { + return nil, fmt.Errorf("failed to create %s headless service. %+v", name, err) + } + return svc, nil +} + +// create a apps.daemonset +func CreateDaemonSet(name, namespace string, clientset kubernetes.Interface, ds *apps.DaemonSet) error { + _, err := clientset.AppsV1beta2().DaemonSets(namespace).Create(ds) + if err != nil { + if k8serrors.IsAlreadyExists(err) { + _, err = clientset.AppsV1beta2().DaemonSets(namespace).Update(ds) + } + if err != nil { + return fmt.Errorf("failed to start %s daemonset: %v\n%v", name, err, ds) + } + } + return err +} + +// create a apps.statefulset and a headless svc +func CreateStatefulSet(name, namespace, appName string, clientset kubernetes.Interface, ss *apps.StatefulSet) (*corev1.Service, error) { + label := ss.GetLabels() + svc, err := makeHeadlessSvc(appName, namespace, label, clientset) + if err != nil { + return nil, fmt.Errorf("failed to start %s service: %v\n%v", name, err, ss) + } + + _, err = clientset.AppsV1beta2().StatefulSets(namespace).Create(ss) + if err != nil { + if k8serrors.IsAlreadyExists(err) { + _, err = clientset.AppsV1beta2().StatefulSets(namespace).Update(ss) + } + if err != nil { + return nil, fmt.Errorf("failed to start %s statefulset: %v\n%v", name, err, ss) + } + } + return svc, err +} diff --git a/tests/framework/installer/ceph_csi_templates.go b/tests/framework/installer/ceph_csi_templates.go new file mode 100644 index 000000000..789678710 --- /dev/null +++ b/tests/framework/installer/ceph_csi_templates.go @@ -0,0 +1,467 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +const ( + rbdAttacherTemplate = ` + kind: StatefulSet + apiVersion: apps/v1beta2 + metadata: + name: csi-rbdplugin-attacher + namespace: {{ .Namespace }} + spec: + serviceName: csi-rbdplugin-attacher + replicas: 1 + selector: + matchLabels: + app: csi-rbdplugin-attacher + template: + metadata: + labels: + app: csi-rbdplugin-attacher + spec: + serviceAccount: rook-csi-rbd-attacher-sa + containers: + - name: csi-rbdplugin-attacher + image: {{ .AttacherImage }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-rbdplugin + type: DirectoryOrCreate +` + rbdProvisionerTemplate = ` + kind: StatefulSet + apiVersion: apps/v1beta2 + metadata: + name: csi-rbdplugin-provisioner + namespace: {{ .Namespace }} + spec: + serviceName: "csi-rbdplugin-provisioner" + replicas: 1 + selector: + matchLabels: + app: csi-rbdplugin-provisioner + template: + metadata: + labels: + app: csi-rbdplugin-provisioner + spec: + serviceAccount: rook-csi-rbd-provisioner-sa + containers: + - name: csi-provisioner + image: {{ .ProvisionerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + - name: csi-snapshotter + image: {{ .SnapshotterImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--connection-timeout=15s" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + imagePullPolicy: Always + securityContext: + privileged: true + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + image: {{ .RBDPluginImage }} + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=csi-rbdplugin" + - "--containerized=true" + - "--metadatastorage=k8s_configmap" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + volumes: + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-rbdplugin + type: DirectoryOrCreate +` + rbdPluginTemplate = ` + kind: DaemonSet + apiVersion: apps/v1beta2 + metadata: + name: csi-rbdplugin + namespace: {{ .Namespace }} + spec: + selector: + matchLabels: + app: csi-rbdplugin + template: + metadata: + labels: + app: csi-rbdplugin + spec: + serviceAccount: rook-csi-rbd-plugin-sa + hostNetwork: true + hostPID: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: {{ .RegistrarImage }} + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: {{ .RBDPluginImage }} + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=csi-rbdplugin" + - "--containerized=true" + - "--metadatastorage=k8s_configmap" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: plugin-mount-dir + mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/ + mountPropagation: "Bidirectional" + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-rbdplugin + type: DirectoryOrCreate + - name: plugin-mount-dir + hostPath: + path: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + ` + cephfsProvisionerTemplate = ` + kind: StatefulSet + apiVersion: apps/v1beta2 + metadata: + name: csi-cephfsplugin-provisioner + namespace: {{ .Namespace }} + spec: + serviceName: "csi-cephfsplugin-provisioner" + replicas: 1 + selector: + matchLabels: + app: csi-cephfsplugin-provisioner + template: + metadata: + labels: + app: csi-cephfsplugin-provisioner + spec: + serviceAccount: rook-csi-cephfs-provisioner-sa + containers: + - name: csi-provisioner + image: {{ .ProvisionerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + image: {{ .CephFSPluginImage }} + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=csi-cephfsplugin" + - "--metadatastorage=k8s_configmap" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-cephfsplugin + type: DirectoryOrCreate + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev +` + cephfsPluginTemplate = ` + kind: DaemonSet + apiVersion: apps/v1beta2 + metadata: + name: csi-cephfsplugin + namespace: {{ .Namespace }} + spec: + selector: + matchLabels: + app: csi-cephfsplugin + template: + metadata: + labels: + app: csi-cephfsplugin + spec: + serviceAccount: rook-csi-cephfs-plugin-sa + hostNetwork: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: {{ .RegistrarImage }} + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-cephfsplugin /registration/csi-cephfsplugin-reg.sock"] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: {{ .CephFSPluginImage }} + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=csi-cephfsplugin" + - "--metadatastorage=k8s_configmap" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + - name: csi-plugins-dir + mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi + mountPropagation: "Bidirectional" + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-cephfsplugin/ + type: DirectoryOrCreate + - name: csi-plugins-dir + hostPath: + path: /var/lib/kubelet/plugins/kubernetes.io/csi + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev +` +) diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index cd22275ce..532e7ec66 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -394,6 +394,21 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(helmInstalled bool, systemNa h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-global", nil) h.k8shelper.Clientset.RbacV1beta1().Roles(systemNamespace).Delete("rook-ceph-system", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-attacher-role", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-external-attacher-runner", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-nodeplugin", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-csi-nodeplugin", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-provisioner-role", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-external-provisioner-runner", nil) + + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("cephfs-csi-nodeplugin", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-csi-nodeplugin", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("cephfs-csi-provisioner-role", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-external-provisioner-runner", nil) + + h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-rbd-config", nil) + h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-cephfs-config", nil) + logger.Infof("done removing the operator from namespace %s", systemNamespace) logger.Infof("removing host data dir %s", h.hostPathToDelete) // removing data dir if exists diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index f84474e6e..99b04a602 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -246,6 +246,7 @@ rules: resources: - pods - configmaps + - services verbs: - get - list @@ -265,6 +266,19 @@ rules: - create - update - delete +- apiGroups: + - apps + resources: + - daemonsets + - statefulsets + verbs: + - get + - list + - watch + - create + - update + - delete + --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole @@ -444,6 +458,250 @@ subjects: name: rook-ceph-system namespace: ` + namespace + ` --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: csi-rbd-config + namespace: ` + namespace + ` +data: + csi-rbdplugin-attacher.yaml: |-` + rbdAttacherTemplate + ` + csi-rbdplugin-provisioner.yaml: |-` + rbdProvisionerTemplate + ` + csi-rbdplugin.yaml: |-` + rbdPluginTemplate + ` +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: csi-cephfs-config + namespace: ` + namespace + ` +data: + csi-cephfsplugin-provisioner.yaml: |` + cephfsProvisionerTemplate + ` + csi-cephfsplugin.yaml: |` + cephfsPluginTemplate + ` +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-attacher-sa + namespace: ` + namespace + ` +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-attacher-runner +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-attacher-role +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-attacher-sa + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: rbd-external-attacher-runner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-plugin-sa + namespace: ` + namespace + ` + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-provisioner-sa + namespace: ` + namespace + ` + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "create", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: rbd-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-plugin-sa + namespace: ` + namespace + ` +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: cephfs-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-provisioner-sa + namespace: ` + namespace + ` +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: cephfs-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- apiVersion: apps/v1beta1 kind: Deployment metadata: @@ -482,7 +740,37 @@ spec: - name: POD_NAMESPACE valueFrom: fieldRef: - fieldPath: metadata.namespace` + fieldPath: metadata.namespace + # CSI enablement + - name: ROOK_CSI_ENABLE_CEPHFS + value: "true" + - name: ROOK_CSI_CEPHFS_IMAGE + value: "quay.io/cephcsi/cephfsplugin:v1.0.0" + - name: ROOK_CSI_ENABLE_RBD + value: "true" + - name: ROOK_CSI_RBD_IMAGE + value: "quay.io/cephcsi/rbdplugin:v1.0.0" + - name: ROOK_CSI_REGISTRAR_IMAGE + value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2" + - name: ROOK_CSI_PROVISIONER_IMAGE + value: "quay.io/k8scsi/csi-provisioner:v1.0.1" + - name: ROOK_CSI_SNAPSHOTTER_IMAGE + value: "quay.io/k8scsi/csi-snapshotter:v1.0.1" + - name: ROOK_CSI_ATTACHER_IMAGE + value: "quay.io/k8scsi/csi-attacher:v1.0.1" + volumeMounts: + - mountPath: /etc/ceph-csi/rbd + name: csi-rbd-config + - mountPath: /etc/ceph-csi/cephfs + name: csi-cephfs-config + volumes: + - name: csi-rbd-config + configMap: + name: csi-rbd-config + - name: csi-cephfs-config + configMap: + name: csi-cephfs-config +` } // GetClusterRoles returns rook-cluster manifest diff --git a/tests/integration/ceph_csi_test.go b/tests/integration/ceph_csi_test.go new file mode 100644 index 000000000..4dec6367f --- /dev/null +++ b/tests/integration/ceph_csi_test.go @@ -0,0 +1,195 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "encoding/json" + "strings" + + monclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + csiSecretName = "ceph-csi-secret" + csiSCRBD = "ceph-csi-rbd" + csiSCCephFS = "ceph-csi-cephfs" + csiPoolRBD = "csi-rbd" + csiPoolCephFS = "csi-cephfs" + csiTestRBDPodName = "csi-test-rbd" + csiTestCephFSPodName = "csi-test-cephfs" +) + +func runCephCSIE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) { + if isCSIRunnig(k8sh, namespace) { + logger.Info("test Ceph CSI driver") + createCephCSISecret(helper, k8sh, s, namespace) + createCephPools(helper, s, namespace) + createCSIStorageClass(k8sh, s, namespace) + createCSIRBDTestPod(k8sh, s, namespace) + } +} + +func isCSIRunnig(k8sh *utils.K8sHelper, namespace string) bool { + return k8sh.IsPodWithLabelPresent("app=csi-rbdplugin", installer.SystemNamespace(namespace)) +} + +func createCephCSISecret(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) { + commandArgs := []string{"-c", "ceph auth get-key client.admin"} + keyResult, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs) + logger.Infof("Ceph get-key: %s", keyResult) + require.Nil(s.T(), err) + commandArgs = []string{"-c", "ceph mon_status"} + monResult, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs) + logger.Infof("Ceph mon_status: %s", monResult) + require.Nil(s.T(), err) + + var mon monclient.MonStatusResponse + err = json.Unmarshal([]byte(monResult), &mon) + require.Nil(s.T(), err) + require.True(s.T(), len(mon.MonMap.Mons) > 0, "no mon found") + monStr := strings.Split(mon.MonMap.Mons[0].Address, "/")[0] + require.True(s.T(), len(monStr) > 0, "invalid mon addr") + + _, err = k8sh.Clientset.Core().Secrets(namespace).Create(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: csiSecretName, + Namespace: namespace, + }, + Data: map[string][]byte{ + "admin": []byte(keyResult), + "adminID": []byte("admin"), + "adminKey": []byte(keyResult), + "monitors": []byte(monStr), + }, + }) + require.Nil(s.T(), err) + logger.Info("Created Ceph CSI Secret") +} + +func createCephPools(helper *clients.TestClient, s suite.Suite, namespace string) { + out, err := helper.PoolClient.Create(csiPoolRBD, namespace, 1) + logger.Infof("rbd pool create: %+v", out) + require.Nil(s.T(), err) + + out, err = helper.PoolClient.Create(csiPoolCephFS, namespace, 1) + logger.Infof("cephFS pool create: %+v", out) + require.Nil(s.T(), err) +} + +func createCSIStorageClass(k8sh *utils.K8sHelper, s suite.Suite, namespace string) { + rbdSC := ` +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ` + csiSCRBD + ` +provisioner: csi-rbdplugin +parameters: + monValueFromSecret: "monitors" + pool: ` + csiPoolRBD + ` + csi.storage.k8s.io/provisioner-secret-name: ` + csiSecretName + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-publish-secret-name: ` + csiSecretName + ` + csi.storage.k8s.io/node-publish-secret-namespace: ` + namespace + ` + adminid: admin + userid: admin +` + cephFSSC := ` +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ` + csiSCCephFS + ` +provisioner: csi-cephFSplugin +parameters: + monValueFromSecret: "monitors" + pool: ` + csiPoolCephFS + ` + csi.storage.k8s.io/provisioner-secret-name: ` + csiSecretName + ` + csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + ` + csi.storage.k8s.io/node-stage-secret-name: ` + csiSecretName + ` + csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + ` + adminid: admin + userid: admin +` + out, err := k8sh.ResourceOperation("apply", rbdSC) + logger.Infof("create rbd storage class: %v", out) + require.Nil(s.T(), err) + + out, err = k8sh.ResourceOperation("apply", cephFSSC) + logger.Infof("create cephfs storage class: %v", out) + require.Nil(s.T(), err) +} + +func createCSIRBDTestPod(k8sh *utils.K8sHelper, s suite.Suite, namespace string) { + pod := ` +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rbd-pvc-csi + namespace: ` + namespace + ` +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: ` + csiSCRBD + ` +--- +apiVersion: v1 +kind: Pod +metadata: + name: ` + csiTestRBDPodName + ` + namespace: ` + namespace + ` +spec: + containers: + - name: ` + csiTestRBDPodName + ` + image: busybox + command: + - sh + - "-c" + - "touch /test/csi.test && sleep 3600" + imagePullPolicy: IfNotPresent + env: + volumeMounts: + - mountPath: /test + name: csivol + volumes: + - name: csivol + persistentVolumeClaim: + claimName: rbd-pvc-csi + readOnly: false + restartPolicy: Never +` + out, err := k8sh.ResourceOperation("create", pod) + logger.Infof("create csi rbd test pod: %v", out) + require.Nil(s.T(), err) + isPodRunning := k8sh.IsPodRunning(csiTestRBDPodName, namespace) + if !isPodRunning { + k8sh.PrintPodDescribe(namespace, csiTestRBDPodName) + k8sh.PrintPodStatus(namespace) + } else { + // cleanup the pod and pv + _, err = k8sh.ResourceOperation("delete", pod) + } + + require.True(s.T(), isPodRunning, "csi rbd test pod fails to run") +} diff --git a/tests/integration/smoke_test.go b/tests/integration/smoke_test.go index 0bba8da48..c4b8d1f9a 100644 --- a/tests/integration/smoke_test.go +++ b/tests/integration/smoke_test.go @@ -81,6 +81,10 @@ func (suite *SmokeSuite) TearDownSuite() { suite.op.Teardown() } +func (suite *SmokeSuite) TestBlockCSI_SmokeTest() { + runCephCSIE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace) +} + func (suite *SmokeSuite) TestBlockStorage_SmokeTest() { runBlockE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace) }