add ceph csi deploy options

Signed-off-by: Huamin Chen <hchen@redhat.com>
This commit is contained in:
Huamin Chen committed 2019-02-12 21:20:41 -05:00
1 parent 78a3995dd0
commit 3806baffe1
33 files changed
+2882 -118

No files matched your search

+79 -113
View File
@@ -4,9 +4,9 @@ weight: 3200
indent: true
---
# Running ceph CSI drivers with Rook
# Running Ceph CSI drivers with Rook
Here is a guide on how to implement ceph-csi drivers on a Kubernetes cluster with Rook. The way to implement it should change relatively soon as there is an effort to manage deployments of the ceph-csi plugin within Rook directly ([#1385](https://github.com/rook/rook/issues/1385), [#2059](https://github.com/rook/rook/pull/2059)).
Here is a guide on how to use Rook to deploy ceph-csi drivers on a Kubernetes cluster.
- [Enable CSI drivers](#enable-csi-drivers)
- [Test the CSI driver](#test-the-csi-driver)
@@ -15,134 +15,112 @@ Here is a guide on how to implement ceph-csi drivers on a Kubernetes cluster wit
## Prerequisites
1. A Kubernetes v1.12+ cluster with at least one node
1. a Kubernetes v1.13+ is needed in order to support CSI Spec 1.0.
2. `--allow-privileged` flag set to true in [kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) and your [API server](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/)
3. An up and running Rook instance (see [Rook - ceph quickstart guide](https://github.com/rook/rook/blob/master/Documentation/ceph-quickstart.md))
3. An up and running Rook instance (see [Rook - Ceph quickstart guide](https://github.com/rook/rook/blob/master/Documentation/ceph-quickstart.md))
## Enable CSI drivers
## CSI Drivers Enablement
As described by the ceph-csi [Cephfs](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-cephfs.md) and [RBD](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-rbd.md) plugins deployment guides, the plugins yaml manifests are located in ceph-csi repository in the [deploy folder](https://github.com/ceph/ceph-csi/tree/master/deploy), and should be deployed as follow:
### Create RBAC used by CSI drivers in the same namespace as Rook Ceph Operator
### Deploy RBACs for node plugins, csi-attacher and csi-provisioner (common to Cephfs and RBD):
```
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-attacher-rbac.yaml
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml
```
Those manifests deploy service accounts, cluster roles and cluster role bindings.
### Deploy csi-attacher and csi-provisioner containers:
Deploys stateful sets for external-attacher and external-provisioner sidecar containers.
##### Cephfs
```
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
```
##### RBD
```
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
```
### Deploy the CSI driver:
Deploys a daemon set with two containers: CSI driver-registrar and the driver.
##### Cephfs
```
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml
```
##### RBD
```
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin.yaml
```console
kubectl create namespace rook-ceph-system
# create rbac. Since rook operator is not permitted to create rbac rules, these rules have to be created outside of operator
kubectl apply -f cluster/examples/kubernetes/ceph/csi/rbac/rbd/
kubectl apply -f cluster/examples/kubernetes/ceph/csi/rbac/cephfs/
```
### Verify the plugin, its attacher and provisioner are running:
You should see an output similar to this when you run `kubectl get all`.
### Create CSI driver deployment templates and persist them in configmaps
```console
kubectl create configmap csi-cephfs-config -n rook-ceph-system --from-file=cluster/examples/kubernetes/ceph/csi/template/cephfs
kubectl create configmap csi-rbd-config -n rook-ceph-system --from-file=cluster/examples/kubernetes/ceph/csi/template/rbd
```
# kubectl get all
NAMESPACE NAME READY STATUS RESTARTS AGE
default pod/csi-rbdplugin-975c4 2/2 Running 0 132m
default pod/csi-rbdplugin-attacher-0 1/1 Running 0 132m
default pod/csi-rbdplugin-provisioner-0 1/1 Running 0 132m
### Start Rook Ceph Operator
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default service/csi-rbdplugin-attacher ClusterIP 10.107.65.227 <none> 12345/TCP 132m
default service/csi-rbdplugin-provisioner ClusterIP 10.108.120.159 <none> 12345/TCP 132m
```console
kubectl apply -f cluster/examples/kubernetes/ceph/operator-with-csi.yaml
```
NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
default daemonset.apps/csi-rbdplugin 1 1 1 1 1 <none> 132m
### Verify CSI drivers and Operator are up and running
NAMESPACE NAME DESIRED CURRENT AGE
default statefulset.apps/csi-rbdplugin-attacher 1 1 132m
default statefulset.apps/csi-rbdplugin-provisioner 1 1 132m
```bash
# kubectl get all -n rook-ceph-system
NAME READY STATUS RESTARTS AGE
pod/csi-cephfsplugin-h5spd 2/2 Running 0 1d
pod/csi-cephfsplugin-provisioner-0 2/2 Running 0 1d
pod/csi-rbdplugin-4l6zg 2/2 Running 2 1d
pod/csi-rbdplugin-attacher-0 1/1 Running 0 1d
pod/csi-rbdplugin-provisioner-0 2/2 Running 2 1d
pod/rook-ceph-agent-zlm84 1/1 Running 0 1d
pod/rook-ceph-operator-c84954957-jdzk6 1/1 Running 0 1d
pod/rook-discover-66hjp 1/1 Running 0 1d
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/csi-cephfsplugin-provisioner ClusterIP 10.0.0.107 <none> 1234/TCP 1d
service/csi-rbdplugin-attacher ClusterIP 10.0.0.109 <none> 1234/TCP 1d
service/csi-rbdplugin-provisioner ClusterIP 10.0.0.56 <none> 1234/TCP 1d
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
daemonset.apps/csi-cephfsplugin 1 1 1 1 1 <none> 1d
daemonset.apps/csi-rbdplugin 1 1 1 1 1 <none> 1d
daemonset.apps/rook-ceph-agent 1 1 1 1 1 <none> 1d
daemonset.apps/rook-discover 1 1 1 1 1 <none> 1d
NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
deployment.apps/rook-ceph-operator 1 1 1 1 1d
NAME DESIRED CURRENT READY AGE
replicaset.apps/rook-ceph-operator-c84954957 1 1 1 1d
NAME DESIRED CURRENT AGE
statefulset.apps/csi-cephfsplugin-provisioner 1 1 1d
statefulset.apps/csi-rbdplugin-attacher 1 1 1d
statefulset.apps/csi-rbdplugin-provisioner 1 1 1d
```
## Test the CSI driver
Once the plugin is successfully deployed, test it by running the following example
Once the plugin is successfully deployed, test it by running the following example.
This example is based on the ceph-csi [examples](https://github.com/ceph/ceph-csi/tree/master/examples) directory. It will describe how to test the RBD csi-driver, but it is very similar to run the cephfs one.
### Create the StorageClass
### Create the StorageClass:
This storageclass expect a pool named `rbd` in your ceph cluster. You can create this pool using [rook pool CRD](https://github.com/rook/rook/blob/master/Documentation/ceph-pool-crd.md)
##### storageclass.yaml
This [storageclass](../cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml) expect a pool named `rbd` in your Ceph cluster. You can create this pool using [rook pool CRD](https://github.com/rook/rook/blob/master/Documentation/ceph-pool-crd.md).
Please update `monitors` to reflect the Ceph mointors.
### Create the Secret
Create a Secret that matches that specified in the [storageclass](../cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml).
Find a Ceph mon pod (in the following example, the pod is `rook-ceph-mon-a-6c4f9f6b6-rzp6r`) and create a Ceph user for that pool called `kubernetes`:
```bash
kubectl exec -ti -n rook-ceph rook-ceph-mon-a-6c4f9f6b6-rzp6r -- bash -c "ceph -c /var/lib/rook/rook-ceph/rook-ceph.config auth get-or-create-key client.kub2 mon \"allow profile rbd\" osd \"profile rbd pool=rbd\""
```
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-rbd
provisioner: csi-rbdplugin
parameters:
# Comma separated list of Ceph monitors
# if using FQDN, make sure csi plugin's dns policy is appropriate.
monitors: mon1:port,mon2:port,...
# if "monitors" parameter is not set, driver to get monitors from same
# secret as admin/user credentials. "monValueFromSecret" provides the
# key in the secret whose value is the mons
#monValueFromSecret: "monitors"
Then create a Secret using admin and `kubernetes` keyrings:
# Ceph pool into which the RBD image shall be created
pool: rbd
# RBD image format. Defaults to "2".
imageFormat: "2"
# RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature.
imageFeatures: layering
# The secrets have to contain Ceph admin credentials.
csiProvisionerSecretName: csi-rbd-secret
csiProvisionerSecretNamespace: default
csiNodePublishSecretName: csi-rbd-secret
csiNodePublishSecretNamespace: default
# Ceph users for operating RBD
adminid: admin
userid: kubernetes
# uncomment the following to use rbd-nbd as mounter on supported nodes
#mounter: rbd-nbd
reclaimPolicy: Delete
```
### Create the Secret:
##### secret.yaml
```
apiVersion: v1
kind: Secret
metadata:
name: csi-rbd-secret
namespace: default
namespace: default
data:
# Key value corresponds to a user name defined in ceph cluster
# Key value corresponds to a user name defined in Ceph cluster
admin: BASE64-ENCODED-PASSWORD
# Key value corresponds to a user name defined in ceph cluster
# Key value corresponds to a user name defined in Ceph cluster
kubernetes: BASE64-ENCODED-PASSWORD
# if monValueFromSecret is set to "monitors", uncomment the
# following and set the mon there
#monitors: BASE64-ENCODED-Comma-Delimited-Mons
```
Here, you need your ceph admin/user password encoded in base64. Run `ceph auth ls` in one of your ceph pod, encode the key of your admin/user and replace `BASE64-ENCODED-PASSWORD` by your encoded key.
Here, you need your Ceph admin/user password encoded in base64. Run `ceph auth ls` in one of your Ceph pod, encode the key of your admin/user and replace `BASE64-ENCODED-PASSWORD` by your encoded key.
### Create the PersistentVolumeClaim:
##### pvc.yaml
```
@@ -188,7 +166,7 @@ spec:
readOnly: false
```
When running `rbd list block --pool [yourpool]` in one of your ceph pod you should see the created PVC:
When running `rbd list block --pool [yourpool]` in one of your Ceph pod you should see the created PVC:
```
# rbd list block --pool rbd
pvc-c20495c0d5de11e8
@@ -220,7 +198,7 @@ You must download this file and modify it to match your Ceph cluster.
```
# wget https://raw.githubusercontent.com/ceph/ceph-csi/master/examples/rbd/snapshotclass.yaml
```
The `csiSnapshotterSecretName` parameter should reference the name of the secret created for the ceph-csi plugin you deployed. The monitors are a comma separated list of your ceph monitors, same as in the StorageClass of the plugin you chosen. When this is done, run:
The `csiSnapshotterSecretName` parameter should reference the name of the secret created for the ceph-csi plugin you deployed. The monitors are a comma separated list of your Ceph monitors, same as in the StorageClass of the plugin you chosen. When this is done, run:
```
# kubectl create -f snapshotclass.yaml
```
@@ -238,7 +216,7 @@ csi-rbdplugin-snapclass 4s
NAME AGE
rbd-pvc-snapshot 6s
```
In one of your ceph pod, run `rbd snap ls [name-of-your-pvc]`.
In one of your Ceph pod, run `rbd snap ls [name-of-your-pvc]`.
The output should be similar to this:
```
# rbd snap ls pvc-c20495c0d5de11e8
@@ -256,20 +234,8 @@ To clean your cluster of the resources created by this example, run the followin
# kubectl delete -f pvc.yaml
# kubectl delete -f secret.yaml
# kubectl delete -f storageclass.yaml
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin.yaml
# Or https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml if you deployed cephfs plugin
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
# Or https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml
# Or https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-attacher-rbac.yaml
```
If you tested snapshots too:
```
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/examples/rbd/snapshot.yaml
+1
View File
@@ -17,6 +17,7 @@
- Recursive chown for mounts can now be toggled with the [ROOK_ENABLE_FSGROUP](https://github.com/rook/rook/issues/2254) environment variable.
- Added the dashboard `port` configuration setting.
- Added the dashboard `ssl` configuration setting.
- Added Ceph CSI driver deployments on Kubernetes 1.13 and above.
## Breaking Changes
@@ -0,0 +1,37 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-cephfs
provisioner: csi-cephfsplugin
parameters:
# Comma separated list of Ceph monitors
# if using FQDN, make sure csi plugin's dns policy is appropriate.
monitors: mon1:port,mon2:port,...
# For provisionVolume: "true":
# A new volume will be created along with a new Ceph user.
# Requires admin credentials (adminID, adminKey).
# For provisionVolume: "false":
# It is assumed the volume already exists and the user is expected
# to provide path to that volume (rootPath) and user credentials (userID, userKey).
provisionVolume: "true"
# Ceph pool into which the volume shall be created
# Required for provisionVolume: "true"
pool: cephfs_data
# Root path of an existing CephFS volume
# Required for provisionVolume: "false"
# rootPath: /absolute/path
# The secrets have to contain user and/or Ceph admin credentials.
csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
csi.storage.k8s.io/provisioner-secret-namespace: default
csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
csi.storage.k8s.io/node-stage-secret-namespace: default
# (optional) The driver can use either ceph-fuse (fuse) or ceph kernel client (kernel)
# If omitted, default volume mounter will be used - this is determined by probing for ceph-fuse
# or by setting the default mounter explicitly via --volumemounter command-line argument.
# mounter: kernel
reclaimPolicy: Delete
@@ -0,0 +1,11 @@
---
apiVersion: snapshot.storage.k8s.io/v1alpha1
kind: VolumeSnapshotClass
metadata:
name: csi-rbdplugin-snapclass
snapshotter: csi-rbdplugin
parameters:
pool: rbd
monitors: mon1:port,mon2:port,...
csi.storage.k8s.io/snapshotter-secret-name: csi-rbd-secret
csi.storage.k8s.io/snapshotter-secret-namespace: default
@@ -0,0 +1,37 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-rbd
provisioner: csi-rbdplugin
parameters:
# Comma separated list of Ceph monitors
# if using FQDN, make sure csi plugin's dns policy is appropriate.
monitors: mon1:port,mon2:port,...
# if "monitors" parameter is not set, driver to get monitors from same
# secret as admin/user credentials. "monValueFromSecret" provides the
# key in the secret whose value is the mons
#monValueFromSecret: "monitors"
# Ceph pool into which the RBD image shall be created
pool: rbd
# RBD image format. Defaults to "2".
imageFormat: "2"
# RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature.
imageFeatures: layering
# The secrets have to contain Ceph admin credentials.
csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
csi.storage.k8s.io/provisioner-secret-namespace: default
csi.storage.k8s.io/node-publish-secret-name: csi-rbd-secret
csi.storage.k8s.io/node-publish-secret-namespace: default
# Ceph users for operating RBD
adminid: admin
userid: kubernetes
# uncomment the following to use rbd-nbd as mounter on supported nodes
#mounter: rbd-nbd
reclaimPolicy: Delete
@@ -0,0 +1,40 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-cephfs-plugin-sa
namespace: rook-ceph-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-nodeplugin
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-plugin-sa
namespace: rook-ceph-system
roleRef:
kind: ClusterRole
name: cephfs-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,43 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-cephfs-provisioner-sa
namespace: rook-ceph-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-external-provisioner-runner
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-provisioner-sa
namespace: rook-ceph-system
roleRef:
kind: ClusterRole
name: cephfs-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,38 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-rbd-attacher-sa
namespace: rook-ceph-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-external-attacher-runner
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-attacher-role
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-attacher-sa
namespace: rook-ceph-system
roleRef:
kind: ClusterRole
name: rbd-external-attacher-runner
apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,41 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-rbd-plugin-sa
namespace: rook-ceph-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-plugin-sa
namespace: rook-ceph-system
roleRef:
kind: ClusterRole
name: rbd-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,59 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-rbd-provisioner-sa
namespace: rook-ceph-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-external-provisioner-runner
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "create", "update"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-provisioner-sa
namespace: rook-ceph-system
roleRef:
kind: ClusterRole
name: rbd-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,78 @@
kind: StatefulSet
apiVersion: apps/v1beta2
metadata:
name: csi-cephfsplugin-provisioner
namespace: {{ .Namespace }}
spec:
serviceName: "csi-cephfsplugin-provisioner"
replicas: 1
selector:
matchLabels:
app: csi-cephfsplugin-provisioner
template:
metadata:
labels:
app: csi-cephfsplugin-provisioner
spec:
serviceAccount: rook-csi-cephfs-provisioner-sa
containers:
- name: csi-provisioner
image: {{ .ProvisionerImage }}
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
- name: csi-cephfsplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
image: {{ .CephFSPluginImage }}
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=csi-cephfsplugin"
- "--metadatastorage=k8s_configmap"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
- name: host-sys
mountPath: /sys
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: host-dev
mountPath: /dev
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-cephfsplugin
type: DirectoryOrCreate
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: host-dev
hostPath:
path: /dev
@@ -0,0 +1,107 @@
kind: DaemonSet
apiVersion: apps/v1beta2
metadata:
name: csi-cephfsplugin
namespace: {{ .Namespace }}
spec:
selector:
matchLabels:
app: csi-cephfsplugin
template:
metadata:
labels:
app: csi-cephfsplugin
spec:
serviceAccount: rook-csi-cephfs-plugin-sa
hostNetwork: true
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
# resolved through k8s service, set dns policy to cluster first
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: driver-registrar
image: {{ .RegistrarImage }}
args:
- "--v=5"
- "--csi-address=/csi/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock"
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "rm -rf /registration/csi-cephfsplugin /registration/csi-cephfsplugin-reg.sock"]
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: csi-cephfsplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: {{ .CephFSPluginImage }}
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=csi-cephfsplugin"
- "--metadatastorage=k8s_configmap"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: plugin-dir
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
- name: csi-plugins-dir
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
mountPropagation: "Bidirectional"
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: host-sys
mountPath: /sys
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: host-dev
mountPath: /dev
volumes:
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-cephfsplugin/
type: DirectoryOrCreate
- name: csi-plugins-dir
hostPath:
path: /var/lib/kubelet/plugins/kubernetes.io/csi
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: host-dev
hostPath:
path: /dev
@@ -0,0 +1,35 @@
kind: StatefulSet
apiVersion: apps/v1beta2
metadata:
name: csi-rbdplugin-attacher
namespace: {{ .Namespace }}
spec:
serviceName: csi-rbdplugin-attacher
replicas: 1
selector:
matchLabels:
app: csi-rbdplugin-attacher
template:
metadata:
labels:
app: csi-rbdplugin-attacher
spec:
serviceAccount: rook-csi-rbd-attacher-sa
containers:
- name: csi-rbdplugin-attacher
image: {{ .AttacherImage }}
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate
@@ -0,0 +1,101 @@
kind: StatefulSet
apiVersion: apps/v1beta2
metadata:
name: csi-rbdplugin-provisioner
namespace: {{ .Namespace }}
spec:
serviceName: "csi-rbdplugin-provisioner"
replicas: 1
selector:
matchLabels:
app: csi-rbdplugin-provisioner
template:
metadata:
labels:
app: csi-rbdplugin-provisioner
spec:
serviceAccount: rook-csi-rbd-provisioner-sa
containers:
- name: csi-provisioner
image: {{ .ProvisionerImage }}
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
- name: csi-snapshotter
image: {{ .SnapshotterImage }}
args:
- "--csi-address=$(ADDRESS)"
- "--connection-timeout=15s"
- "--v=5"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
imagePullPolicy: Always
securityContext:
privileged: true
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
- name: csi-rbdplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
image: {{ .RBDPluginImage }}
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=csi-rbdplugin"
- "--containerized=true"
- "--metadatastorage=k8s_configmap"
env:
- name: HOST_ROOTFS
value: "/rootfs"
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
- mountPath: /dev
name: host-dev
- mountPath: /rootfs
name: host-rootfs
- mountPath: /sys
name: host-sys
- mountPath: /lib/modules
name: lib-modules
readOnly: true
volumes:
- name: host-dev
hostPath:
path: /dev
- name: host-rootfs
hostPath:
path: /
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate
@@ -0,0 +1,116 @@
kind: DaemonSet
apiVersion: apps/v1beta2
metadata:
name: csi-rbdplugin
namespace: {{ .Namespace }}
spec:
selector:
matchLabels:
app: csi-rbdplugin
template:
metadata:
labels:
app: csi-rbdplugin
spec:
serviceAccount: rook-csi-rbd-plugin-sa
hostNetwork: true
hostPID: true
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
# resolved through k8s service, set dns policy to cluster first
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: driver-registrar
image: {{ .RegistrarImage }}
args:
- "--v=5"
- "--csi-address=/csi/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock"
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"]
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: csi-rbdplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: {{ .RBDPluginImage }}
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=csi-rbdplugin"
- "--containerized=true"
- "--metadatastorage=k8s_configmap"
env:
- name: HOST_ROOTFS
value: "/rootfs"
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: plugin-dir
mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: plugin-mount-dir
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
mountPropagation: "Bidirectional"
- mountPath: /dev
name: host-dev
- mountPath: /rootfs
name: host-rootfs
- mountPath: /sys
name: host-sys
- mountPath: /lib/modules
name: lib-modules
readOnly: true
volumes:
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate
- name: plugin-mount-dir
hostPath:
path: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- name: host-dev
hostPath:
path: /dev
- name: host-rootfs
hostPath:
path: /
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
+538
View File
@@ -0,0 +1,538 @@
apiVersion: v1
kind: Namespace
metadata:
name: rook-ceph-system
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephclusters.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephCluster
listKind: CephClusterList
plural: cephclusters
singular: cephcluster
scope: Namespaced
version: v1
validation:
openAPIV3Schema:
properties:
spec:
properties:
cephVersion:
properties:
allowUnsupported:
type: boolean
image:
type: string
name:
pattern: ^(luminous|mimic|nautilus)$
type: string
dashboard:
properties:
enabled:
type: boolean
urlPrefix:
type: string
port:
type: integer
dataDirHostPath:
pattern: ^/(\S+)
type: string
mon:
properties:
allowMultiplePerNode:
type: boolean
count:
maximum: 9
minimum: 1
type: integer
required:
- count
network:
properties:
hostNetwork:
type: boolean
storage:
properties:
nodes:
items: {}
type: array
useAllDevices: {}
useAllNodes:
type: boolean
required:
- mon
additionalPrinterColumns:
- name: DataDirHostPath
type: string
description: Directory used on the K8s nodes
JSONPath: .spec.dataDirHostPath
- name: MonCount
type: string
description: Number of MONs
JSONPath: .spec.mon.count
- name: Age
type: date
JSONPath: .metadata.creationTimestamp
- name: State
type: string
description: Current State
JSONPath: .status.state
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephfilesystems.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephFilesystem
listKind: CephFilesystemList
plural: cephfilesystems
singular: cephfilesystem
scope: Namespaced
version: v1
additionalPrinterColumns:
- name: MdsCount
type: string
description: Number of MDSs
JSONPath: .spec.metadataServer.activeCount
- name: Age
type: date
JSONPath: .metadata.creationTimestamp
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephnfses.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephNFS
listKind: CephNFSList
plural: cephnfses
singular: cephnfs
shortNames:
- nfs
scope: Namespaced
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephobjectstores.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephObjectStore
listKind: CephObjectStoreList
plural: cephobjectstores
singular: cephobjectstore
scope: Namespaced
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephobjectstoreusers.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephObjectStoreUser
listKind: CephObjectStoreUserList
plural: cephobjectstoreusers
singular: cephobjectstoreuser
scope: Namespaced
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephblockpools.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephBlockPool
listKind: CephBlockPoolList
plural: cephblockpools
singular: cephblockpool
scope: Namespaced
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: volumes.rook.io
spec:
group: rook.io
names:
kind: Volume
listKind: VolumeList
plural: volumes
singular: volume
shortNames:
- rv
scope: Namespaced
version: v1alpha2
---
# The cluster role for managing all the cluster-specific resources in a namespace
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-cluster-mgmt
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
- secrets
- pods
- pods/log
- services
- configmaps
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- extensions
resources:
- deployments
- daemonsets
- replicasets
verbs:
- get
- list
- watch
- create
- update
- delete
---
# The role for the operator to manage resources in the system namespace
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: rook-ceph-system
namespace: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
- services
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- extensions
resources:
- daemonsets
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- apps
resources:
- daemonsets
- statefulsets
verbs:
- get
- list
- watch
- create
- update
- delete
---
# The cluster role for managing the Rook CRDs
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-global
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
# Pod access is needed for fencing
- pods
# Node access is needed for determining nodes where mons should run
- nodes
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
# PVs and PVCs are managed by the Rook provisioner
- persistentvolumes
- persistentvolumeclaims
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- batch
resources:
- jobs
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- ceph.rook.io
resources:
- "*"
verbs:
- "*"
- apiGroups:
- rook.io
resources:
- "*"
verbs:
- "*"
---
# Aspects of ceph-mgr that require cluster-wide access
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-mgr-cluster
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
- configmaps
- nodes
- nodes/proxy
verbs:
- get
- list
- watch
---
# The rook system service account used by the operator, agent, and discovery pods
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-system
namespace: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
---
# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-system
namespace: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-system
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: rook-ceph-system
---
# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-global
namespace: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-global
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: rook-ceph-system
---
# The deployment for the rook operator
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: rook-ceph-operator
namespace: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
spec:
replicas: 1
template:
metadata:
labels:
app: rook-ceph-operator
spec:
serviceAccountName: rook-ceph-system
containers:
- name: rook-ceph-operator
image: rook/ceph:master
args: ["ceph", "operator"]
volumeMounts:
- mountPath: /var/lib/rook
name: rook-config
- mountPath: /etc/ceph
name: default-config-dir
- mountPath: /etc/ceph-csi/rbd
name: csi-rbd-config
- mountPath: /etc/ceph-csi/cephfs
name: csi-cephfs-config
env:
# To disable RBAC, uncomment the following:
# - name: RBAC_ENABLED
# value: "false"
# Rook Agent toleration. Will tolerate all taints with all keys.
# Choose between NoSchedule, PreferNoSchedule and NoExecute:
# - name: AGENT_TOLERATION
# value: "NoSchedule"
# (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate
# - name: AGENT_TOLERATION_KEY
# value: "<KeyOfTheTaintToTolerate>"
# (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`.
# `Any` uses Ceph admin credentials by default/fallback.
# For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and
# set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name.
# to the namespace in which the `mountSecret` Kubernetes secret namespace.
# - name: AGENT_MOUNT_SECURITY_MODE
# value: "Any"
# Set the path where the Rook agent can find the flex volumes
# - name: FLEXVOLUME_DIR_PATH
# value: "<PathToFlexVolumes>"
# Set the path where kernel modules can be found
# - name: LIB_MODULES_DIR_PATH
# value: "<PathToLibModules>"
# Mount any extra directories into the agent container
# - name: AGENT_MOUNTS
# value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2"
# Rook Discover toleration. Will tolerate all taints with all keys.
# Choose between NoSchedule, PreferNoSchedule and NoExecute:
# - name: DISCOVER_TOLERATION
# value: "NoSchedule"
# (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate
# - name: DISCOVER_TOLERATION_KEY
# value: "<KeyOfTheTaintToTolerate>"
# Allow rook to create multiple file systems. Note: This is considered
# an experimental feature in Ceph as described at
# http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster
# which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
- name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
value: "false"
# The logging level for the operator: INFO | DEBUG
- name: ROOK_LOG_LEVEL
value: "INFO"
# The interval to check if every mon is in the quorum.
- name: ROOK_MON_HEALTHCHECK_INTERVAL
value: "45s"
# The duration to wait before trying to failover or remove/replace the
# current mon with a new mon (useful for compensating flapping network).
- name: ROOK_MON_OUT_TIMEOUT
value: "300s"
# The duration between discovering devices in the rook-discover daemonset.
- name: ROOK_DISCOVER_DEVICES_INTERVAL
value: "60m"
# Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods.
# This is necessary to workaround the anyuid issues when running on OpenShift.
# For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
- name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
value: "false"
# In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins).
# Disable it here if you have similiar issues.
# For more details see https://github.com/rook/rook/issues/2417
- name: ROOK_ENABLE_SELINUX_RELABELING
value: "true"
# CSI enablement
- name: ROOK_CSI_ENABLE_CEPHFS
value: "true"
- name: ROOK_CSI_CEPHFS_IMAGE
value: "quay.io/cephcsi/cephfsplugin:v1.0.0"
- name: ROOK_CSI_ENABLE_RBD
value: "true"
- name: ROOK_CSI_RBD_IMAGE
value: "quay.io/cephcsi/rbdplugin:v1.0.0"
- name: ROOK_CSI_REGISTRAR_IMAGE
value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2"
- name: ROOK_CSI_PROVISIONER_IMAGE
value: "quay.io/k8scsi/csi-provisioner:v1.0.1"
- name: ROOK_CSI_SNAPSHOTTER_IMAGE
value: "quay.io/k8scsi/csi-snapshotter:v1.0.1"
- name: ROOK_CSI_ATTACHER_IMAGE
value: "quay.io/k8scsi/csi-attacher:v1.0.1"
# The name of the node to pass with the downward API
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
# The pod name to pass with the downward API
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
# The pod namespace to pass with the downward API
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumes:
- name: rook-config
emptyDir: {}
- name: default-config-dir
emptyDir: {}
- name: csi-rbd-config
configMap:
name: csi-rbd-config
- name: csi-cephfs-config
configMap:
name: csi-cephfs-config
+20
View File
@@ -24,6 +24,7 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
"github.com/rook/rook/pkg/operator/ceph"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
@@ -42,6 +43,25 @@ https://github.com/rook/rook`,
func init() {
operatorCmd.Flags().DurationVar(&mon.HealthCheckInterval, "mon-healthcheck-interval", mon.HealthCheckInterval, "mon health check interval (duration)")
operatorCmd.Flags().DurationVar(&mon.MonOutTimeout, "mon-out-timeout", mon.MonOutTimeout, "mon out timeout (duration)")
operatorCmd.Flags().BoolVar(&csi.EnableRBD, "csi-enable-rbd", false, "whether enable ceph-csi rbd driver")
operatorCmd.Flags().BoolVar(&csi.EnableCephFS, "csi-enable-cephfs", false, "whether enable ceph-csi cephfs driver")
// csi images
operatorCmd.Flags().StringVar(&csi.CSIParam.RBDPluginImage, "csi-rbd-image", csi.DefaultRBDPluginImage, "ceph-csi rbd plugin image")
operatorCmd.Flags().StringVar(&csi.CSIParam.CephFSPluginImage, "csi-cephfs-image", csi.DefaultCephFSPluginImage, "ceph-csi cephfs plugin image")
operatorCmd.Flags().StringVar(&csi.CSIParam.RegistrarImage, "csi-registrar-image", csi.DefaultRegistrarImage, "csi registrar image")
operatorCmd.Flags().StringVar(&csi.CSIParam.ProvisionerImage, "csi-provisioner-image", csi.DefaultProvisionerImage, "csi provisioner image")
operatorCmd.Flags().StringVar(&csi.CSIParam.AttacherImage, "csi-attacher-image", csi.DefaultAttacherImage, "csi attacher image")
operatorCmd.Flags().StringVar(&csi.CSIParam.SnapshotterImage, "csi-snapshotter-image", csi.DefaultSnapshotterImage, "csi snapshotter image")
// csi deployment templates
operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template")
operatorCmd.Flags().StringVar(&csi.RBDProvisionerTemplatePath, "csi-rbd-provisioner-template-path", csi.DefaultRBDProvisionerTemplatePath, "path to ceph-csi rbd provisioner template")
operatorCmd.Flags().StringVar(&csi.RBDAttacherTemplatePath, "csi-rbd-attacher-template-path", csi.DefaultRBDAttacherTemplatePath, "path to ceph-csi rbd attacher template")
operatorCmd.Flags().StringVar(&csi.CephFSPluginTemplatePath, "csi-cephfs-plugin-template-path", csi.DefaultCephFSPluginTemplatePath, "path to ceph-csi cephfs plugin template")
operatorCmd.Flags().StringVar(&csi.CephFSProvisionerTemplatePath, "csi-cephfs-provisioner-template-path", csi.DefaultCephFSProvisionerTemplatePath, "path to ceph-csi cephfs provisioner template")
flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix)
flags.SetLoggingFlags(operatorCmd.Flags())
operatorCmd.RunE = startOperator
@@ -0,0 +1 @@
../../../../cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml
+1
View File
@@ -0,0 +1 @@
../../../../cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
+1
View File
@@ -0,0 +1 @@
../../../../cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml
+1
View File
@@ -0,0 +1 @@
../../../../cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml
+1
View File
@@ -0,0 +1 @@
../../../../cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml
+198
View File
@@ -0,0 +1,198 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package csi
import (
"fmt"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1beta2"
"k8s.io/client-go/kubernetes"
)
type Param struct {
Namespace string
RBDPluginImage string
CephFSPluginImage string
RegistrarImage string
ProvisionerImage string
AttacherImage string
SnapshotterImage string
}
var (
CSIParam Param
EnableRBD = true
EnableCephFS = true
// template paths
RBDPluginTemplatePath string
RBDProvisionerTemplatePath string
RBDAttacherTemplatePath string
CephFSPluginTemplatePath string
CephFSProvisionerTemplatePath string
)
const (
KubeMinMajor = "1"
KubeMinMinor = "13"
// image names
DefaultRBDPluginImage = "quay.io/cephcsi/rbdplugin:v1.0.0"
DefaultCephFSPluginImage = "quay.io/cephcsi/cephfsplugin:v1.0.0"
DefaultRegistrarImage = "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2"
DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.0.1"
DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v1.0.1"
DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.0.1"
// template
DefaultRBDPluginTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml"
DefaultRBDProvisionerTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml"
DefaultRBDAttacherTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-attacher.yaml"
DefaultCephFSPluginTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml"
DefaultCephFSProvisionerTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner.yaml"
ExitOnError = false // don't exit if CSI fails to deploy. Switch to true when flexdriver is disabled
)
func CSIEnabled() bool {
if EnableRBD || EnableCephFS {
return true
}
return false
}
func SetCSINamespace(namespace string) {
CSIParam.Namespace = namespace
}
func ValidateCSIParam() error {
if EnableRBD {
if len(CSIParam.RBDPluginImage) == 0 {
return fmt.Errorf("missing csi rbd plugin image")
}
if len(CSIParam.RegistrarImage) == 0 {
return fmt.Errorf("missing csi registrar image")
}
if len(CSIParam.ProvisionerImage) == 0 {
return fmt.Errorf("missing csi provisioner image")
}
if len(CSIParam.AttacherImage) == 0 {
return fmt.Errorf("missing csi attacher image")
}
if len(RBDPluginTemplatePath) == 0 {
return fmt.Errorf("missing rbd plugin template path")
}
if len(RBDProvisionerTemplatePath) == 0 {
return fmt.Errorf("missing rbd provisioner template path")
}
if len(RBDAttacherTemplatePath) == 0 {
return fmt.Errorf("missing rbd attacher template path")
}
}
if EnableCephFS {
if len(CSIParam.CephFSPluginImage) == 0 {
return fmt.Errorf("missing csi cephfs plugin image")
}
if len(CSIParam.RegistrarImage) == 0 {
return fmt.Errorf("missing csi registrar image")
}
if len(CSIParam.ProvisionerImage) == 0 {
return fmt.Errorf("missing csi provisioner image")
}
if len(CephFSPluginTemplatePath) == 0 {
return fmt.Errorf("missing cephfs plugin template path")
}
if len(CephFSProvisionerTemplatePath) == 0 {
return fmt.Errorf("missing ceph provisioner template path")
}
}
return nil
}
func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
var (
err error
rbdPlugin, cephfsPlugin *apps.DaemonSet
rbdProvisioner, rbdAttacher, cephfsProvisioner *apps.StatefulSet
)
if EnableRBD {
rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath)
if err != nil {
return fmt.Errorf("failed to load rbd plugin template: %v", err)
}
rbdProvisioner, err = templateToStatefulSet("rbd-provisioner", RBDProvisionerTemplatePath)
if err != nil {
return fmt.Errorf("failed to load rbd provisioner template: %v", err)
}
rbdAttacher, err = templateToStatefulSet("rbd-attacher", RBDAttacherTemplatePath)
if err != nil {
return fmt.Errorf("failed to load rbd attacher template: %v", err)
}
}
if EnableCephFS {
cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath)
if err != nil {
return fmt.Errorf("failed to load CephFS plugin template: %v", err)
}
cephfsProvisioner, err = templateToStatefulSet("cephfs-provisioner", CephFSProvisionerTemplatePath)
if err != nil {
return fmt.Errorf("failed to load CephFS provisioner template: %v", err)
}
}
if rbdPlugin != nil {
err = k8sutil.CreateDaemonSet("csi rbd plugin", namespace, clientset, rbdPlugin)
if err != nil {
return fmt.Errorf("failed to start rbdplugin daemonset: %v\n%v", err, rbdPlugin)
}
}
if rbdProvisioner != nil {
_, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisioner)
if err != nil {
return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisioner)
}
}
if rbdAttacher != nil {
_, err = k8sutil.CreateStatefulSet("csi rbd attacher", namespace, "csi-rbdplugin-attacher", clientset, rbdAttacher)
if err != nil {
return fmt.Errorf("failed to start rbd attacher statefulset: %v\n%v", err, rbdAttacher)
}
}
if cephfsPlugin != nil {
err = k8sutil.CreateDaemonSet("csi cephfs plugin", namespace, clientset, cephfsPlugin)
if err != nil {
return fmt.Errorf("failed to start cephfs plugin daemonset: %v\n%v", err, cephfsPlugin)
}
}
if cephfsProvisioner != nil {
_, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisioner)
if err != nil {
return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisioner)
}
}
return nil
}
+46
View File
@@ -0,0 +1,46 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package csi
import (
"testing"
"github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
)
func TestStartCSI(t *testing.T) {
RBDPluginTemplatePath = "csi-rbdplugin.yaml"
RBDProvisionerTemplatePath = "csi-rbdplugin-provisioner.yaml"
RBDAttacherTemplatePath = "csi-rbdplugin-attacher.yaml"
CephFSPluginTemplatePath = "csi-cephfsplugin.yaml"
CephFSProvisionerTemplatePath = "csi-cephfsplugin-provisioner.yaml"
CSIParam = Param{
RBDPluginImage: "image",
CephFSPluginImage: "image",
RegistrarImage: "image",
ProvisionerImage: "image",
AttacherImage: "image",
SnapshotterImage: "image",
}
clientset := test.New(3)
err := StartCSIDrivers("ns", clientset)
assert.Nil(t, err)
}
+67
View File
@@ -0,0 +1,67 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package csi
import (
"bytes"
"io/ioutil"
"text/template"
"github.com/ghodss/yaml"
apps "k8s.io/api/apps/v1beta2"
)
func loadTemplate(name, templatePath string) (string, error) {
b, err := ioutil.ReadFile(templatePath)
if err != nil {
return "", err
}
data := string(b)
var writer bytes.Buffer
t := template.New(name)
err = template.Must(t.Parse(data)).Execute(&writer, CSIParam)
return writer.String(), err
}
func templateToStatefulSet(name, templatePath string) (*apps.StatefulSet, error) {
var ss apps.StatefulSet
t, err := loadTemplate(name, templatePath)
if err != nil {
return nil, err
}
err = yaml.Unmarshal([]byte(t), &ss)
if err != nil {
return nil, err
}
return &ss, nil
}
func templateToDaemonSet(name, templatePath string) (*apps.DaemonSet, error) {
var ds apps.DaemonSet
t, err := loadTemplate(name, templatePath)
if err != nil {
return nil, err
}
err = yaml.Unmarshal([]byte(t), &ds)
if err != nil {
return nil, err
}
return &ds, nil
}
+108
View File
@@ -0,0 +1,108 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package csi
import (
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/assert"
)
var (
testDSTemplate = []byte(`
kind: DaemonSet
apiVersion: apps/v1beta2
metadata:
name: test-label
namespace: {{ .Namespace }}
spec:
selector:
matchLabels:
app: test-label
template:
metadata:
labels:
app: test-label
spec:
serviceAccount: test-sa
containers:
- name: registrar
image: {{ .RegistrarImage }}
- name: rbdplugin
image: {{ .RBDPluginImage }}
- name: cephfsplugin
image: {{ .CephFSPluginImage }}
`)
testSSTemplate = []byte(`
kind: StatefulSet
apiVersion: apps/v1beta2
metadata:
name: test-label
namespace: {{ .Namespace }}
spec:
selector:
matchLabels:
app: test-label
template:
metadata:
labels:
app: test-label
spec:
serviceAccount: test-sa
containers:
- name: csi-rbdplugin-attacher
image: {{ .AttacherImage }}
- name: provisioner
image: {{ .ProvisionerImage }}
- name: rbdplugin
image: {{ .RBDPluginImage }}
- name: cephfsplugin
image: {{ .CephFSPluginImage }}
`)
)
func TestDaemonSetTemplate(t *testing.T) {
tmp, err := ioutil.TempFile("", "yaml")
assert.Nil(t, err)
defer os.Remove(tmp.Name())
_, err = tmp.Write(testDSTemplate)
assert.Nil(t, err)
err = tmp.Close()
assert.Nil(t, err)
_, err = templateToDaemonSet("test-ds", tmp.Name())
assert.Nil(t, err)
}
func TestStatefulSetTemplate(t *testing.T) {
tmp, err := ioutil.TempFile("", "yaml")
assert.Nil(t, err)
defer os.Remove(tmp.Name())
_, err = tmp.Write(testSSTemplate)
assert.Nil(t, err)
err = tmp.Close()
assert.Nil(t, err)
_, err = templateToStatefulSet("test-ss", tmp.Name())
assert.Nil(t, err)
}
+25 -4
View File
@@ -31,6 +31,7 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
"github.com/rook/rook/pkg/operator/ceph/agent"
"github.com/rook/rook/pkg/operator/ceph/cluster"
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/ceph/file"
"github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/pkg/operator/ceph/object/user"
@@ -100,15 +101,35 @@ func (o *Operator) Run() error {
return fmt.Errorf("Error starting device discovery daemonset: %v", err)
}
signalChan := make(chan os.Signal, 1)
stopChan := make(chan struct{})
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
serverVersion, err := o.context.Clientset.Discovery().ServerVersion()
if err != nil {
return fmt.Errorf("Error getting server version: %v", err)
}
if serverVersion.Major >= csi.KubeMinMajor && serverVersion.Minor >= csi.KubeMinMinor && csi.CSIEnabled() {
logger.Infof("Ceph CSI driver is enabled, validate csi param")
if err = csi.ValidateCSIParam(); err != nil {
logger.Warningf("invalid csi params: %v", err)
if csi.ExitOnError {
return err
}
} else {
csi.SetCSINamespace(namespace)
if err = csi.StartCSIDrivers(namespace, o.context.Clientset); err != nil {
logger.Warningf("failed to start Ceph csi drivers: %v", err)
if csi.ExitOnError {
return err
}
} else {
logger.Infof("successfully started Ceph csi drivers")
}
}
}
signalChan := make(chan os.Signal, 1)
stopChan := make(chan struct{})
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
// Run volume provisioner for each of the supported configurations
for name, vendor := range provisionerConfigs {
volumeProvisioner := provisioner.New(o.context, vendor)
+82
View File
@@ -0,0 +1,82 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8sutil
import (
"fmt"
apps "k8s.io/api/apps/v1beta2"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
// make a headless svc for statefulset
func makeHeadlessSvc(name, namespace string, label map[string]string, clientset kubernetes.Interface) (*corev1.Service, error) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: label,
},
Spec: corev1.ServiceSpec{
Selector: label,
Ports: []corev1.ServicePort{{Name: "dummy", Port: 1234}},
},
}
_, err := clientset.CoreV1().Services(namespace).Create(svc)
if err != nil && !k8serrors.IsAlreadyExists(err) {
return nil, fmt.Errorf("failed to create %s headless service. %+v", name, err)
}
return svc, nil
}
// create a apps.daemonset
func CreateDaemonSet(name, namespace string, clientset kubernetes.Interface, ds *apps.DaemonSet) error {
_, err := clientset.AppsV1beta2().DaemonSets(namespace).Create(ds)
if err != nil {
if k8serrors.IsAlreadyExists(err) {
_, err = clientset.AppsV1beta2().DaemonSets(namespace).Update(ds)
}
if err != nil {
return fmt.Errorf("failed to start %s daemonset: %v\n%v", name, err, ds)
}
}
return err
}
// create a apps.statefulset and a headless svc
func CreateStatefulSet(name, namespace, appName string, clientset kubernetes.Interface, ss *apps.StatefulSet) (*corev1.Service, error) {
label := ss.GetLabels()
svc, err := makeHeadlessSvc(appName, namespace, label, clientset)
if err != nil {
return nil, fmt.Errorf("failed to start %s service: %v\n%v", name, err, ss)
}
_, err = clientset.AppsV1beta2().StatefulSets(namespace).Create(ss)
if err != nil {
if k8serrors.IsAlreadyExists(err) {
_, err = clientset.AppsV1beta2().StatefulSets(namespace).Update(ss)
}
if err != nil {
return nil, fmt.Errorf("failed to start %s statefulset: %v\n%v", name, err, ss)
}
}
return svc, err
}
@@ -0,0 +1,467 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
const (
rbdAttacherTemplate = `
kind: StatefulSet
apiVersion: apps/v1beta2
metadata:
name: csi-rbdplugin-attacher
namespace: {{ .Namespace }}
spec:
serviceName: csi-rbdplugin-attacher
replicas: 1
selector:
matchLabels:
app: csi-rbdplugin-attacher
template:
metadata:
labels:
app: csi-rbdplugin-attacher
spec:
serviceAccount: rook-csi-rbd-attacher-sa
containers:
- name: csi-rbdplugin-attacher
image: {{ .AttacherImage }}
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate
`
rbdProvisionerTemplate = `
kind: StatefulSet
apiVersion: apps/v1beta2
metadata:
name: csi-rbdplugin-provisioner
namespace: {{ .Namespace }}
spec:
serviceName: "csi-rbdplugin-provisioner"
replicas: 1
selector:
matchLabels:
app: csi-rbdplugin-provisioner
template:
metadata:
labels:
app: csi-rbdplugin-provisioner
spec:
serviceAccount: rook-csi-rbd-provisioner-sa
containers:
- name: csi-provisioner
image: {{ .ProvisionerImage }}
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
- name: csi-snapshotter
image: {{ .SnapshotterImage }}
args:
- "--csi-address=$(ADDRESS)"
- "--connection-timeout=15s"
- "--v=5"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
imagePullPolicy: Always
securityContext:
privileged: true
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
- name: csi-rbdplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
image: {{ .RBDPluginImage }}
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=csi-rbdplugin"
- "--containerized=true"
- "--metadatastorage=k8s_configmap"
env:
- name: HOST_ROOTFS
value: "/rootfs"
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
- mountPath: /dev
name: host-dev
- mountPath: /rootfs
name: host-rootfs
- mountPath: /sys
name: host-sys
- mountPath: /lib/modules
name: lib-modules
readOnly: true
volumes:
- name: host-dev
hostPath:
path: /dev
- name: host-rootfs
hostPath:
path: /
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate
`
rbdPluginTemplate = `
kind: DaemonSet
apiVersion: apps/v1beta2
metadata:
name: csi-rbdplugin
namespace: {{ .Namespace }}
spec:
selector:
matchLabels:
app: csi-rbdplugin
template:
metadata:
labels:
app: csi-rbdplugin
spec:
serviceAccount: rook-csi-rbd-plugin-sa
hostNetwork: true
hostPID: true
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
# resolved through k8s service, set dns policy to cluster first
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: driver-registrar
image: {{ .RegistrarImage }}
args:
- "--v=5"
- "--csi-address=/csi/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock"
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"]
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: csi-rbdplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: {{ .RBDPluginImage }}
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=csi-rbdplugin"
- "--containerized=true"
- "--metadatastorage=k8s_configmap"
env:
- name: HOST_ROOTFS
value: "/rootfs"
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: plugin-dir
mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: plugin-mount-dir
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
mountPropagation: "Bidirectional"
- mountPath: /dev
name: host-dev
- mountPath: /rootfs
name: host-rootfs
- mountPath: /sys
name: host-sys
- mountPath: /lib/modules
name: lib-modules
readOnly: true
volumes:
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate
- name: plugin-mount-dir
hostPath:
path: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- name: host-dev
hostPath:
path: /dev
- name: host-rootfs
hostPath:
path: /
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
`
cephfsProvisionerTemplate = `
kind: StatefulSet
apiVersion: apps/v1beta2
metadata:
name: csi-cephfsplugin-provisioner
namespace: {{ .Namespace }}
spec:
serviceName: "csi-cephfsplugin-provisioner"
replicas: 1
selector:
matchLabels:
app: csi-cephfsplugin-provisioner
template:
metadata:
labels:
app: csi-cephfsplugin-provisioner
spec:
serviceAccount: rook-csi-cephfs-provisioner-sa
containers:
- name: csi-provisioner
image: {{ .ProvisionerImage }}
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
- name: csi-cephfsplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
image: {{ .CephFSPluginImage }}
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=csi-cephfsplugin"
- "--metadatastorage=k8s_configmap"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
- name: host-sys
mountPath: /sys
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: host-dev
mountPath: /dev
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-cephfsplugin
type: DirectoryOrCreate
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: host-dev
hostPath:
path: /dev
`
cephfsPluginTemplate = `
kind: DaemonSet
apiVersion: apps/v1beta2
metadata:
name: csi-cephfsplugin
namespace: {{ .Namespace }}
spec:
selector:
matchLabels:
app: csi-cephfsplugin
template:
metadata:
labels:
app: csi-cephfsplugin
spec:
serviceAccount: rook-csi-cephfs-plugin-sa
hostNetwork: true
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
# resolved through k8s service, set dns policy to cluster first
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: driver-registrar
image: {{ .RegistrarImage }}
args:
- "--v=5"
- "--csi-address=/csi/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock"
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "rm -rf /registration/csi-cephfsplugin /registration/csi-cephfsplugin-reg.sock"]
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: csi-cephfsplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: {{ .CephFSPluginImage }}
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=csi-cephfsplugin"
- "--metadatastorage=k8s_configmap"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: plugin-dir
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
- name: csi-plugins-dir
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
mountPropagation: "Bidirectional"
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: host-sys
mountPath: /sys
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: host-dev
mountPath: /dev
volumes:
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-cephfsplugin/
type: DirectoryOrCreate
- name: csi-plugins-dir
hostPath:
path: /var/lib/kubelet/plugins/kubernetes.io/csi
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: host-dev
hostPath:
path: /dev
`
)
@@ -394,6 +394,21 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-global", nil)
h.k8shelper.Clientset.RbacV1beta1().Roles(systemNamespace).Delete("rook-ceph-system", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-attacher-role", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-external-attacher-runner", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-nodeplugin", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-csi-nodeplugin", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-provisioner-role", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-external-provisioner-runner", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("cephfs-csi-nodeplugin", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-csi-nodeplugin", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("cephfs-csi-provisioner-role", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-external-provisioner-runner", nil)
h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-rbd-config", nil)
h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-cephfs-config", nil)
logger.Infof("done removing the operator from namespace %s", systemNamespace)
logger.Infof("removing host data dir %s", h.hostPathToDelete)
// removing data dir if exists
+289 -1
View File
@@ -246,6 +246,7 @@ rules:
resources:
- pods
- configmaps
- services
verbs:
- get
- list
@@ -265,6 +266,19 @@ rules:
- create
- update
- delete
- apiGroups:
- apps
resources:
- daemonsets
- statefulsets
verbs:
- get
- list
- watch
- create
- update
- delete
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
@@ -444,6 +458,250 @@ subjects:
name: rook-ceph-system
namespace: ` + namespace + `
---
apiVersion: v1
kind: ConfigMap
metadata:
name: csi-rbd-config
namespace: ` + namespace + `
data:
csi-rbdplugin-attacher.yaml: |-` + rbdAttacherTemplate + `
csi-rbdplugin-provisioner.yaml: |-` + rbdProvisionerTemplate + `
csi-rbdplugin.yaml: |-` + rbdPluginTemplate + `
---
apiVersion: v1
kind: ConfigMap
metadata:
name: csi-cephfs-config
namespace: ` + namespace + `
data:
csi-cephfsplugin-provisioner.yaml: |` + cephfsProvisionerTemplate + `
csi-cephfsplugin.yaml: |` + cephfsPluginTemplate + `
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-rbd-attacher-sa
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-external-attacher-runner
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-attacher-role
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-attacher-sa
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: rbd-external-attacher-runner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-rbd-plugin-sa
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-plugin-sa
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: rbd-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-rbd-provisioner-sa
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-external-provisioner-runner
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "create", "update"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-provisioner-sa
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: rbd-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-cephfs-plugin-sa
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-nodeplugin
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-plugin-sa
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: cephfs-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-cephfs-provisioner-sa
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-external-provisioner-runner
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-provisioner-sa
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: cephfs-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
@@ -482,7 +740,37 @@ spec:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace`
fieldPath: metadata.namespace
# CSI enablement
- name: ROOK_CSI_ENABLE_CEPHFS
value: "true"
- name: ROOK_CSI_CEPHFS_IMAGE
value: "quay.io/cephcsi/cephfsplugin:v1.0.0"
- name: ROOK_CSI_ENABLE_RBD
value: "true"
- name: ROOK_CSI_RBD_IMAGE
value: "quay.io/cephcsi/rbdplugin:v1.0.0"
- name: ROOK_CSI_REGISTRAR_IMAGE
value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2"
- name: ROOK_CSI_PROVISIONER_IMAGE
value: "quay.io/k8scsi/csi-provisioner:v1.0.1"
- name: ROOK_CSI_SNAPSHOTTER_IMAGE
value: "quay.io/k8scsi/csi-snapshotter:v1.0.1"
- name: ROOK_CSI_ATTACHER_IMAGE
value: "quay.io/k8scsi/csi-attacher:v1.0.1"
volumeMounts:
- mountPath: /etc/ceph-csi/rbd
name: csi-rbd-config
- mountPath: /etc/ceph-csi/cephfs
name: csi-cephfs-config
volumes:
- name: csi-rbd-config
configMap:
name: csi-rbd-config
- name: csi-cephfs-config
configMap:
name: csi-cephfs-config
`
}
// GetClusterRoles returns rook-cluster manifest
+195
View File
@@ -0,0 +1,195 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"encoding/json"
"strings"
monclient "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
csiSecretName = "ceph-csi-secret"
csiSCRBD = "ceph-csi-rbd"
csiSCCephFS = "ceph-csi-cephfs"
csiPoolRBD = "csi-rbd"
csiPoolCephFS = "csi-cephfs"
csiTestRBDPodName = "csi-test-rbd"
csiTestCephFSPodName = "csi-test-cephfs"
)
func runCephCSIE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
if isCSIRunnig(k8sh, namespace) {
logger.Info("test Ceph CSI driver")
createCephCSISecret(helper, k8sh, s, namespace)
createCephPools(helper, s, namespace)
createCSIStorageClass(k8sh, s, namespace)
createCSIRBDTestPod(k8sh, s, namespace)
}
}
func isCSIRunnig(k8sh *utils.K8sHelper, namespace string) bool {
return k8sh.IsPodWithLabelPresent("app=csi-rbdplugin", installer.SystemNamespace(namespace))
}
func createCephCSISecret(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
commandArgs := []string{"-c", "ceph auth get-key client.admin"}
keyResult, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs)
logger.Infof("Ceph get-key: %s", keyResult)
require.Nil(s.T(), err)
commandArgs = []string{"-c", "ceph mon_status"}
monResult, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs)
logger.Infof("Ceph mon_status: %s", monResult)
require.Nil(s.T(), err)
var mon monclient.MonStatusResponse
err = json.Unmarshal([]byte(monResult), &mon)
require.Nil(s.T(), err)
require.True(s.T(), len(mon.MonMap.Mons) > 0, "no mon found")
monStr := strings.Split(mon.MonMap.Mons[0].Address, "/")[0]
require.True(s.T(), len(monStr) > 0, "invalid mon addr")
_, err = k8sh.Clientset.Core().Secrets(namespace).Create(&v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: csiSecretName,
Namespace: namespace,
},
Data: map[string][]byte{
"admin": []byte(keyResult),
"adminID": []byte("admin"),
"adminKey": []byte(keyResult),
"monitors": []byte(monStr),
},
})
require.Nil(s.T(), err)
logger.Info("Created Ceph CSI Secret")
}
func createCephPools(helper *clients.TestClient, s suite.Suite, namespace string) {
out, err := helper.PoolClient.Create(csiPoolRBD, namespace, 1)
logger.Infof("rbd pool create: %+v", out)
require.Nil(s.T(), err)
out, err = helper.PoolClient.Create(csiPoolCephFS, namespace, 1)
logger.Infof("cephFS pool create: %+v", out)
require.Nil(s.T(), err)
}
func createCSIStorageClass(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
rbdSC := `
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ` + csiSCRBD + `
provisioner: csi-rbdplugin
parameters:
monValueFromSecret: "monitors"
pool: ` + csiPoolRBD + `
csi.storage.k8s.io/provisioner-secret-name: ` + csiSecretName + `
csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + `
csi.storage.k8s.io/node-publish-secret-name: ` + csiSecretName + `
csi.storage.k8s.io/node-publish-secret-namespace: ` + namespace + `
adminid: admin
userid: admin
`
cephFSSC := `
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ` + csiSCCephFS + `
provisioner: csi-cephFSplugin
parameters:
monValueFromSecret: "monitors"
pool: ` + csiPoolCephFS + `
csi.storage.k8s.io/provisioner-secret-name: ` + csiSecretName + `
csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + `
csi.storage.k8s.io/node-stage-secret-name: ` + csiSecretName + `
csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + `
adminid: admin
userid: admin
`
out, err := k8sh.ResourceOperation("apply", rbdSC)
logger.Infof("create rbd storage class: %v", out)
require.Nil(s.T(), err)
out, err = k8sh.ResourceOperation("apply", cephFSSC)
logger.Infof("create cephfs storage class: %v", out)
require.Nil(s.T(), err)
}
func createCSIRBDTestPod(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
pod := `
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rbd-pvc-csi
namespace: ` + namespace + `
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: ` + csiSCRBD + `
---
apiVersion: v1
kind: Pod
metadata:
name: ` + csiTestRBDPodName + `
namespace: ` + namespace + `
spec:
containers:
- name: ` + csiTestRBDPodName + `
image: busybox
command:
- sh
- "-c"
- "touch /test/csi.test && sleep 3600"
imagePullPolicy: IfNotPresent
env:
volumeMounts:
- mountPath: /test
name: csivol
volumes:
- name: csivol
persistentVolumeClaim:
claimName: rbd-pvc-csi
readOnly: false
restartPolicy: Never
`
out, err := k8sh.ResourceOperation("create", pod)
logger.Infof("create csi rbd test pod: %v", out)
require.Nil(s.T(), err)
isPodRunning := k8sh.IsPodRunning(csiTestRBDPodName, namespace)
if !isPodRunning {
k8sh.PrintPodDescribe(namespace, csiTestRBDPodName)
k8sh.PrintPodStatus(namespace)
} else {
// cleanup the pod and pv
_, err = k8sh.ResourceOperation("delete", pod)
}
require.True(s.T(), isPodRunning, "csi rbd test pod fails to run")
}
+4
View File
@@ -81,6 +81,10 @@ func (suite *SmokeSuite) TearDownSuite() {
suite.op.Teardown()
}
func (suite *SmokeSuite) TestBlockCSI_SmokeTest() {
runCephCSIE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
}
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {
runBlockE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
}