add ceph csi deploy options
Signed-off-by: Huamin Chen <hchen@redhat.com>
This commit is contained in:
1 parent
78a3995dd0
commit
3806baffe1
33 files changed
+2882
-118
No files matched your search
@@ -4,9 +4,9 @@ weight: 3200
|
||||
indent: true
|
||||
---
|
||||
|
||||
# Running ceph CSI drivers with Rook
|
||||
# Running Ceph CSI drivers with Rook
|
||||
|
||||
Here is a guide on how to implement ceph-csi drivers on a Kubernetes cluster with Rook. The way to implement it should change relatively soon as there is an effort to manage deployments of the ceph-csi plugin within Rook directly ([#1385](https://github.com/rook/rook/issues/1385), [#2059](https://github.com/rook/rook/pull/2059)).
|
||||
Here is a guide on how to use Rook to deploy ceph-csi drivers on a Kubernetes cluster.
|
||||
|
||||
- [Enable CSI drivers](#enable-csi-drivers)
|
||||
- [Test the CSI driver](#test-the-csi-driver)
|
||||
@@ -15,134 +15,112 @@ Here is a guide on how to implement ceph-csi drivers on a Kubernetes cluster wit
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. A Kubernetes v1.12+ cluster with at least one node
|
||||
1. a Kubernetes v1.13+ is needed in order to support CSI Spec 1.0.
|
||||
2. `--allow-privileged` flag set to true in [kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) and your [API server](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/)
|
||||
3. An up and running Rook instance (see [Rook - ceph quickstart guide](https://github.com/rook/rook/blob/master/Documentation/ceph-quickstart.md))
|
||||
3. An up and running Rook instance (see [Rook - Ceph quickstart guide](https://github.com/rook/rook/blob/master/Documentation/ceph-quickstart.md))
|
||||
|
||||
## Enable CSI drivers
|
||||
## CSI Drivers Enablement
|
||||
|
||||
As described by the ceph-csi [Cephfs](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-cephfs.md) and [RBD](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-rbd.md) plugins deployment guides, the plugins yaml manifests are located in ceph-csi repository in the [deploy folder](https://github.com/ceph/ceph-csi/tree/master/deploy), and should be deployed as follow:
|
||||
### Create RBAC used by CSI drivers in the same namespace as Rook Ceph Operator
|
||||
|
||||
### Deploy RBACs for node plugins, csi-attacher and csi-provisioner (common to Cephfs and RBD):
|
||||
```
|
||||
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-attacher-rbac.yaml
|
||||
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
|
||||
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml
|
||||
```
|
||||
Those manifests deploy service accounts, cluster roles and cluster role bindings.
|
||||
### Deploy csi-attacher and csi-provisioner containers:
|
||||
Deploys stateful sets for external-attacher and external-provisioner sidecar containers.
|
||||
##### Cephfs
|
||||
|
||||
```
|
||||
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml
|
||||
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
|
||||
```
|
||||
##### RBD
|
||||
```
|
||||
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml
|
||||
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
|
||||
```
|
||||
### Deploy the CSI driver:
|
||||
Deploys a daemon set with two containers: CSI driver-registrar and the driver.
|
||||
##### Cephfs
|
||||
```
|
||||
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml
|
||||
```
|
||||
##### RBD
|
||||
```
|
||||
kubectl create -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin.yaml
|
||||
```console
|
||||
kubectl create namespace rook-ceph-system
|
||||
# create rbac. Since rook operator is not permitted to create rbac rules, these rules have to be created outside of operator
|
||||
kubectl apply -f cluster/examples/kubernetes/ceph/csi/rbac/rbd/
|
||||
kubectl apply -f cluster/examples/kubernetes/ceph/csi/rbac/cephfs/
|
||||
```
|
||||
|
||||
### Verify the plugin, its attacher and provisioner are running:
|
||||
You should see an output similar to this when you run `kubectl get all`.
|
||||
### Create CSI driver deployment templates and persist them in configmaps
|
||||
|
||||
```console
|
||||
kubectl create configmap csi-cephfs-config -n rook-ceph-system --from-file=cluster/examples/kubernetes/ceph/csi/template/cephfs
|
||||
|
||||
kubectl create configmap csi-rbd-config -n rook-ceph-system --from-file=cluster/examples/kubernetes/ceph/csi/template/rbd
|
||||
```
|
||||
# kubectl get all
|
||||
|
||||
NAMESPACE NAME READY STATUS RESTARTS AGE
|
||||
default pod/csi-rbdplugin-975c4 2/2 Running 0 132m
|
||||
default pod/csi-rbdplugin-attacher-0 1/1 Running 0 132m
|
||||
default pod/csi-rbdplugin-provisioner-0 1/1 Running 0 132m
|
||||
### Start Rook Ceph Operator
|
||||
|
||||
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
default service/csi-rbdplugin-attacher ClusterIP 10.107.65.227 <none> 12345/TCP 132m
|
||||
default service/csi-rbdplugin-provisioner ClusterIP 10.108.120.159 <none> 12345/TCP 132m
|
||||
```console
|
||||
kubectl apply -f cluster/examples/kubernetes/ceph/operator-with-csi.yaml
|
||||
```
|
||||
|
||||
NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
|
||||
default daemonset.apps/csi-rbdplugin 1 1 1 1 1 <none> 132m
|
||||
### Verify CSI drivers and Operator are up and running
|
||||
|
||||
NAMESPACE NAME DESIRED CURRENT AGE
|
||||
default statefulset.apps/csi-rbdplugin-attacher 1 1 132m
|
||||
default statefulset.apps/csi-rbdplugin-provisioner 1 1 132m
|
||||
```bash
|
||||
# kubectl get all -n rook-ceph-system
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
pod/csi-cephfsplugin-h5spd 2/2 Running 0 1d
|
||||
pod/csi-cephfsplugin-provisioner-0 2/2 Running 0 1d
|
||||
pod/csi-rbdplugin-4l6zg 2/2 Running 2 1d
|
||||
pod/csi-rbdplugin-attacher-0 1/1 Running 0 1d
|
||||
pod/csi-rbdplugin-provisioner-0 2/2 Running 2 1d
|
||||
pod/rook-ceph-agent-zlm84 1/1 Running 0 1d
|
||||
pod/rook-ceph-operator-c84954957-jdzk6 1/1 Running 0 1d
|
||||
pod/rook-discover-66hjp 1/1 Running 0 1d
|
||||
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
service/csi-cephfsplugin-provisioner ClusterIP 10.0.0.107 <none> 1234/TCP 1d
|
||||
service/csi-rbdplugin-attacher ClusterIP 10.0.0.109 <none> 1234/TCP 1d
|
||||
service/csi-rbdplugin-provisioner ClusterIP 10.0.0.56 <none> 1234/TCP 1d
|
||||
|
||||
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
|
||||
daemonset.apps/csi-cephfsplugin 1 1 1 1 1 <none> 1d
|
||||
daemonset.apps/csi-rbdplugin 1 1 1 1 1 <none> 1d
|
||||
daemonset.apps/rook-ceph-agent 1 1 1 1 1 <none> 1d
|
||||
daemonset.apps/rook-discover 1 1 1 1 1 <none> 1d
|
||||
|
||||
NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
|
||||
deployment.apps/rook-ceph-operator 1 1 1 1 1d
|
||||
|
||||
NAME DESIRED CURRENT READY AGE
|
||||
replicaset.apps/rook-ceph-operator-c84954957 1 1 1 1d
|
||||
|
||||
NAME DESIRED CURRENT AGE
|
||||
statefulset.apps/csi-cephfsplugin-provisioner 1 1 1d
|
||||
statefulset.apps/csi-rbdplugin-attacher 1 1 1d
|
||||
statefulset.apps/csi-rbdplugin-provisioner 1 1 1d
|
||||
```
|
||||
|
||||
## Test the CSI driver
|
||||
|
||||
Once the plugin is successfully deployed, test it by running the following example
|
||||
Once the plugin is successfully deployed, test it by running the following example.
|
||||
|
||||
This example is based on the ceph-csi [examples](https://github.com/ceph/ceph-csi/tree/master/examples) directory. It will describe how to test the RBD csi-driver, but it is very similar to run the cephfs one.
|
||||
### Create the StorageClass
|
||||
|
||||
### Create the StorageClass:
|
||||
This storageclass expect a pool named `rbd` in your ceph cluster. You can create this pool using [rook pool CRD](https://github.com/rook/rook/blob/master/Documentation/ceph-pool-crd.md)
|
||||
##### storageclass.yaml
|
||||
This [storageclass](../cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml) expect a pool named `rbd` in your Ceph cluster. You can create this pool using [rook pool CRD](https://github.com/rook/rook/blob/master/Documentation/ceph-pool-crd.md).
|
||||
|
||||
Please update `monitors` to reflect the Ceph mointors.
|
||||
|
||||
|
||||
### Create the Secret
|
||||
|
||||
Create a Secret that matches that specified in the [storageclass](../cluster/examples/kubernetes/ceph/csi/example/rbd/storageclass.yaml).
|
||||
|
||||
Find a Ceph mon pod (in the following example, the pod is `rook-ceph-mon-a-6c4f9f6b6-rzp6r`) and create a Ceph user for that pool called `kubernetes`:
|
||||
```bash
|
||||
kubectl exec -ti -n rook-ceph rook-ceph-mon-a-6c4f9f6b6-rzp6r -- bash -c "ceph -c /var/lib/rook/rook-ceph/rook-ceph.config auth get-or-create-key client.kub2 mon \"allow profile rbd\" osd \"profile rbd pool=rbd\""
|
||||
```
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: csi-rbd
|
||||
provisioner: csi-rbdplugin
|
||||
parameters:
|
||||
# Comma separated list of Ceph monitors
|
||||
# if using FQDN, make sure csi plugin's dns policy is appropriate.
|
||||
monitors: mon1:port,mon2:port,...
|
||||
|
||||
# if "monitors" parameter is not set, driver to get monitors from same
|
||||
# secret as admin/user credentials. "monValueFromSecret" provides the
|
||||
# key in the secret whose value is the mons
|
||||
#monValueFromSecret: "monitors"
|
||||
Then create a Secret using admin and `kubernetes` keyrings:
|
||||
|
||||
|
||||
# Ceph pool into which the RBD image shall be created
|
||||
pool: rbd
|
||||
|
||||
# RBD image format. Defaults to "2".
|
||||
imageFormat: "2"
|
||||
|
||||
# RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature.
|
||||
imageFeatures: layering
|
||||
|
||||
# The secrets have to contain Ceph admin credentials.
|
||||
csiProvisionerSecretName: csi-rbd-secret
|
||||
csiProvisionerSecretNamespace: default
|
||||
csiNodePublishSecretName: csi-rbd-secret
|
||||
csiNodePublishSecretNamespace: default
|
||||
|
||||
# Ceph users for operating RBD
|
||||
adminid: admin
|
||||
userid: kubernetes
|
||||
# uncomment the following to use rbd-nbd as mounter on supported nodes
|
||||
#mounter: rbd-nbd
|
||||
reclaimPolicy: Delete
|
||||
```
|
||||
### Create the Secret:
|
||||
##### secret.yaml
|
||||
```
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: csi-rbd-secret
|
||||
namespace: default
|
||||
namespace: default
|
||||
data:
|
||||
# Key value corresponds to a user name defined in ceph cluster
|
||||
# Key value corresponds to a user name defined in Ceph cluster
|
||||
admin: BASE64-ENCODED-PASSWORD
|
||||
# Key value corresponds to a user name defined in ceph cluster
|
||||
# Key value corresponds to a user name defined in Ceph cluster
|
||||
kubernetes: BASE64-ENCODED-PASSWORD
|
||||
# if monValueFromSecret is set to "monitors", uncomment the
|
||||
# following and set the mon there
|
||||
#monitors: BASE64-ENCODED-Comma-Delimited-Mons
|
||||
```
|
||||
Here, you need your ceph admin/user password encoded in base64. Run `ceph auth ls` in one of your ceph pod, encode the key of your admin/user and replace `BASE64-ENCODED-PASSWORD` by your encoded key.
|
||||
|
||||
Here, you need your Ceph admin/user password encoded in base64. Run `ceph auth ls` in one of your Ceph pod, encode the key of your admin/user and replace `BASE64-ENCODED-PASSWORD` by your encoded key.
|
||||
|
||||
### Create the PersistentVolumeClaim:
|
||||
##### pvc.yaml
|
||||
```
|
||||
@@ -188,7 +166,7 @@ spec:
|
||||
readOnly: false
|
||||
```
|
||||
|
||||
When running `rbd list block --pool [yourpool]` in one of your ceph pod you should see the created PVC:
|
||||
When running `rbd list block --pool [yourpool]` in one of your Ceph pod you should see the created PVC:
|
||||
```
|
||||
# rbd list block --pool rbd
|
||||
pvc-c20495c0d5de11e8
|
||||
@@ -220,7 +198,7 @@ You must download this file and modify it to match your Ceph cluster.
|
||||
```
|
||||
# wget https://raw.githubusercontent.com/ceph/ceph-csi/master/examples/rbd/snapshotclass.yaml
|
||||
```
|
||||
The `csiSnapshotterSecretName` parameter should reference the name of the secret created for the ceph-csi plugin you deployed. The monitors are a comma separated list of your ceph monitors, same as in the StorageClass of the plugin you chosen. When this is done, run:
|
||||
The `csiSnapshotterSecretName` parameter should reference the name of the secret created for the ceph-csi plugin you deployed. The monitors are a comma separated list of your Ceph monitors, same as in the StorageClass of the plugin you chosen. When this is done, run:
|
||||
```
|
||||
# kubectl create -f snapshotclass.yaml
|
||||
```
|
||||
@@ -238,7 +216,7 @@ csi-rbdplugin-snapclass 4s
|
||||
NAME AGE
|
||||
rbd-pvc-snapshot 6s
|
||||
```
|
||||
In one of your ceph pod, run `rbd snap ls [name-of-your-pvc]`.
|
||||
In one of your Ceph pod, run `rbd snap ls [name-of-your-pvc]`.
|
||||
The output should be similar to this:
|
||||
```
|
||||
# rbd snap ls pvc-c20495c0d5de11e8
|
||||
@@ -256,20 +234,8 @@ To clean your cluster of the resources created by this example, run the followin
|
||||
# kubectl delete -f pvc.yaml
|
||||
# kubectl delete -f secret.yaml
|
||||
# kubectl delete -f storageclass.yaml
|
||||
|
||||
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin.yaml
|
||||
# Or https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml if you deployed cephfs plugin
|
||||
|
||||
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
|
||||
# Or https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
|
||||
|
||||
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml
|
||||
# Or https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml
|
||||
|
||||
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml
|
||||
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
|
||||
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-attacher-rbac.yaml
|
||||
```
|
||||
|
||||
If you tested snapshots too:
|
||||
```
|
||||
# kubectl delete -f https://raw.githubusercontent.com/ceph/ceph-csi/master/examples/rbd/snapshot.yaml
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
- Recursive chown for mounts can now be toggled with the [ROOK_ENABLE_FSGROUP](https://github.com/rook/rook/issues/2254) environment variable.
|
||||
- Added the dashboard `port` configuration setting.
|
||||
- Added the dashboard `ssl` configuration setting.
|
||||
- Added Ceph CSI driver deployments on Kubernetes 1.13 and above.
|
||||
|
||||
## Breaking Changes
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: csi-cephfs
|
||||
provisioner: csi-cephfsplugin
|
||||
parameters:
|
||||
# Comma separated list of Ceph monitors
|
||||
# if using FQDN, make sure csi plugin's dns policy is appropriate.
|
||||
monitors: mon1:port,mon2:port,...
|
||||
|
||||
# For provisionVolume: "true":
|
||||
# A new volume will be created along with a new Ceph user.
|
||||
# Requires admin credentials (adminID, adminKey).
|
||||
# For provisionVolume: "false":
|
||||
# It is assumed the volume already exists and the user is expected
|
||||
# to provide path to that volume (rootPath) and user credentials (userID, userKey).
|
||||
provisionVolume: "true"
|
||||
|
||||
# Ceph pool into which the volume shall be created
|
||||
# Required for provisionVolume: "true"
|
||||
pool: cephfs_data
|
||||
|
||||
# Root path of an existing CephFS volume
|
||||
# Required for provisionVolume: "false"
|
||||
# rootPath: /absolute/path
|
||||
|
||||
# The secrets have to contain user and/or Ceph admin credentials.
|
||||
csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: default
|
||||
csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: default
|
||||
|
||||
# (optional) The driver can use either ceph-fuse (fuse) or ceph kernel client (kernel)
|
||||
# If omitted, default volume mounter will be used - this is determined by probing for ceph-fuse
|
||||
# or by setting the default mounter explicitly via --volumemounter command-line argument.
|
||||
# mounter: kernel
|
||||
reclaimPolicy: Delete
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
apiVersion: snapshot.storage.k8s.io/v1alpha1
|
||||
kind: VolumeSnapshotClass
|
||||
metadata:
|
||||
name: csi-rbdplugin-snapclass
|
||||
snapshotter: csi-rbdplugin
|
||||
parameters:
|
||||
pool: rbd
|
||||
monitors: mon1:port,mon2:port,...
|
||||
csi.storage.k8s.io/snapshotter-secret-name: csi-rbd-secret
|
||||
csi.storage.k8s.io/snapshotter-secret-namespace: default
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: csi-rbd
|
||||
provisioner: csi-rbdplugin
|
||||
parameters:
|
||||
# Comma separated list of Ceph monitors
|
||||
# if using FQDN, make sure csi plugin's dns policy is appropriate.
|
||||
monitors: mon1:port,mon2:port,...
|
||||
|
||||
# if "monitors" parameter is not set, driver to get monitors from same
|
||||
# secret as admin/user credentials. "monValueFromSecret" provides the
|
||||
# key in the secret whose value is the mons
|
||||
#monValueFromSecret: "monitors"
|
||||
|
||||
|
||||
# Ceph pool into which the RBD image shall be created
|
||||
pool: rbd
|
||||
|
||||
# RBD image format. Defaults to "2".
|
||||
imageFormat: "2"
|
||||
|
||||
# RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature.
|
||||
imageFeatures: layering
|
||||
|
||||
# The secrets have to contain Ceph admin credentials.
|
||||
csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: default
|
||||
csi.storage.k8s.io/node-publish-secret-name: csi-rbd-secret
|
||||
csi.storage.k8s.io/node-publish-secret-namespace: default
|
||||
|
||||
# Ceph users for operating RBD
|
||||
adminid: admin
|
||||
userid: kubernetes
|
||||
# uncomment the following to use rbd-nbd as mounter on supported nodes
|
||||
#mounter: rbd-nbd
|
||||
reclaimPolicy: Delete
|
||||
@@ -0,0 +1,40 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-cephfs-plugin-sa
|
||||
namespace: rook-ceph-system
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: cephfs-csi-nodeplugin
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: cephfs-csi-nodeplugin
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-cephfs-plugin-sa
|
||||
namespace: rook-ceph-system
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: cephfs-csi-nodeplugin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
@@ -0,0 +1,43 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-cephfs-provisioner-sa
|
||||
namespace: rook-ceph-system
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: cephfs-external-provisioner-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "create", "delete"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: cephfs-csi-provisioner-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-cephfs-provisioner-sa
|
||||
namespace: rook-ceph-system
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: cephfs-external-provisioner-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-rbd-attacher-sa
|
||||
namespace: rook-ceph-system
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-external-attacher-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-csi-attacher-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-rbd-attacher-sa
|
||||
namespace: rook-ceph-system
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: rbd-external-attacher-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
@@ -0,0 +1,41 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-rbd-plugin-sa
|
||||
namespace: rook-ceph-system
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-csi-nodeplugin
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-csi-nodeplugin
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-rbd-plugin-sa
|
||||
namespace: rook-ceph-system
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: rbd-csi-nodeplugin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
@@ -0,0 +1,59 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-rbd-provisioner-sa
|
||||
namespace: rook-ceph-system
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-external-provisioner-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["endpoints"]
|
||||
verbs: ["get", "create", "update"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "create", "delete"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "delete"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["apiextensions.k8s.io"]
|
||||
resources: ["customresourcedefinitions"]
|
||||
verbs: ["create"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-csi-provisioner-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-rbd-provisioner-sa
|
||||
namespace: rook-ceph-system
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: rbd-external-provisioner-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-cephfsplugin-provisioner
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
serviceName: "csi-cephfsplugin-provisioner"
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-cephfsplugin-provisioner
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-cephfsplugin-provisioner
|
||||
spec:
|
||||
serviceAccount: rook-csi-cephfs-provisioner-sa
|
||||
containers:
|
||||
- name: csi-provisioner
|
||||
image: {{ .ProvisionerImage }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
|
||||
- name: csi-cephfsplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
image: {{ .CephFSPluginImage }}
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=csi-cephfsplugin"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
|
||||
- name: host-sys
|
||||
mountPath: /sys
|
||||
- name: lib-modules
|
||||
mountPath: /lib/modules
|
||||
readOnly: true
|
||||
- name: host-dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-cephfsplugin
|
||||
type: DirectoryOrCreate
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
@@ -0,0 +1,107 @@
|
||||
kind: DaemonSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-cephfsplugin
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-cephfsplugin
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-cephfsplugin
|
||||
spec:
|
||||
serviceAccount: rook-csi-cephfs-plugin-sa
|
||||
hostNetwork: true
|
||||
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
|
||||
# resolved through k8s service, set dns policy to cluster first
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
containers:
|
||||
- name: driver-registrar
|
||||
image: {{ .RegistrarImage }}
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=/csi/csi.sock"
|
||||
- "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock"
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: ["/bin/sh", "-c", "rm -rf /registration/csi-cephfsplugin /registration/csi-cephfsplugin-reg.sock"]
|
||||
env:
|
||||
- name: KUBE_NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /csi
|
||||
- name: registration-dir
|
||||
mountPath: /registration
|
||||
- name: csi-cephfsplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: {{ .CephFSPluginImage }}
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=csi-cephfsplugin"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
|
||||
- name: csi-plugins-dir
|
||||
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: pods-mount-dir
|
||||
mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: host-sys
|
||||
mountPath: /sys
|
||||
- name: lib-modules
|
||||
mountPath: /lib/modules
|
||||
readOnly: true
|
||||
- name: host-dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: plugin-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-cephfsplugin/
|
||||
type: DirectoryOrCreate
|
||||
- name: csi-plugins-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/kubernetes.io/csi
|
||||
type: DirectoryOrCreate
|
||||
- name: registration-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins_registry/
|
||||
type: Directory
|
||||
- name: pods-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
@@ -0,0 +1,35 @@
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-rbdplugin-attacher
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
serviceName: csi-rbdplugin-attacher
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-rbdplugin-attacher
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-rbdplugin-attacher
|
||||
spec:
|
||||
serviceAccount: rook-csi-rbd-attacher-sa
|
||||
containers:
|
||||
- name: csi-rbdplugin-attacher
|
||||
image: {{ .AttacherImage }}
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
type: DirectoryOrCreate
|
||||
@@ -0,0 +1,101 @@
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-rbdplugin-provisioner
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
serviceName: "csi-rbdplugin-provisioner"
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-rbdplugin-provisioner
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-rbdplugin-provisioner
|
||||
spec:
|
||||
serviceAccount: rook-csi-rbd-provisioner-sa
|
||||
containers:
|
||||
- name: csi-provisioner
|
||||
image: {{ .ProvisionerImage }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
- name: csi-snapshotter
|
||||
image: {{ .SnapshotterImage }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--connection-timeout=15s"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
|
||||
imagePullPolicy: Always
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
- name: csi-rbdplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
image: {{ .RBDPluginImage }}
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=csi-rbdplugin"
|
||||
- "--containerized=true"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /rootfs
|
||||
name: host-rootfs
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- mountPath: /lib/modules
|
||||
name: lib-modules
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: host-rootfs
|
||||
hostPath:
|
||||
path: /
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
type: DirectoryOrCreate
|
||||
@@ -0,0 +1,116 @@
|
||||
kind: DaemonSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-rbdplugin
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-rbdplugin
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-rbdplugin
|
||||
spec:
|
||||
serviceAccount: rook-csi-rbd-plugin-sa
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
|
||||
# resolved through k8s service, set dns policy to cluster first
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
containers:
|
||||
- name: driver-registrar
|
||||
image: {{ .RegistrarImage }}
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=/csi/csi.sock"
|
||||
- "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock"
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"]
|
||||
env:
|
||||
- name: KUBE_NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /csi
|
||||
- name: registration-dir
|
||||
mountPath: /registration
|
||||
- name: csi-rbdplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: {{ .RBDPluginImage }}
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=csi-rbdplugin"
|
||||
- "--containerized=true"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin
|
||||
- name: pods-mount-dir
|
||||
mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: plugin-mount-dir
|
||||
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
|
||||
mountPropagation: "Bidirectional"
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /rootfs
|
||||
name: host-rootfs
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- mountPath: /lib/modules
|
||||
name: lib-modules
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: plugin-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
type: DirectoryOrCreate
|
||||
- name: plugin-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
|
||||
type: DirectoryOrCreate
|
||||
- name: registration-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins_registry/
|
||||
type: Directory
|
||||
- name: pods-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: host-rootfs
|
||||
hostPath:
|
||||
path: /
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
+538
@@ -0,0 +1,538 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: cephclusters.ceph.rook.io
|
||||
spec:
|
||||
group: ceph.rook.io
|
||||
names:
|
||||
kind: CephCluster
|
||||
listKind: CephClusterList
|
||||
plural: cephclusters
|
||||
singular: cephcluster
|
||||
scope: Namespaced
|
||||
version: v1
|
||||
validation:
|
||||
openAPIV3Schema:
|
||||
properties:
|
||||
spec:
|
||||
properties:
|
||||
cephVersion:
|
||||
properties:
|
||||
allowUnsupported:
|
||||
type: boolean
|
||||
image:
|
||||
type: string
|
||||
name:
|
||||
pattern: ^(luminous|mimic|nautilus)$
|
||||
type: string
|
||||
dashboard:
|
||||
properties:
|
||||
enabled:
|
||||
type: boolean
|
||||
urlPrefix:
|
||||
type: string
|
||||
port:
|
||||
type: integer
|
||||
dataDirHostPath:
|
||||
pattern: ^/(\S+)
|
||||
type: string
|
||||
mon:
|
||||
properties:
|
||||
allowMultiplePerNode:
|
||||
type: boolean
|
||||
count:
|
||||
maximum: 9
|
||||
minimum: 1
|
||||
type: integer
|
||||
required:
|
||||
- count
|
||||
network:
|
||||
properties:
|
||||
hostNetwork:
|
||||
type: boolean
|
||||
storage:
|
||||
properties:
|
||||
nodes:
|
||||
items: {}
|
||||
type: array
|
||||
useAllDevices: {}
|
||||
useAllNodes:
|
||||
type: boolean
|
||||
required:
|
||||
- mon
|
||||
additionalPrinterColumns:
|
||||
- name: DataDirHostPath
|
||||
type: string
|
||||
description: Directory used on the K8s nodes
|
||||
JSONPath: .spec.dataDirHostPath
|
||||
- name: MonCount
|
||||
type: string
|
||||
description: Number of MONs
|
||||
JSONPath: .spec.mon.count
|
||||
- name: Age
|
||||
type: date
|
||||
JSONPath: .metadata.creationTimestamp
|
||||
- name: State
|
||||
type: string
|
||||
description: Current State
|
||||
JSONPath: .status.state
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: cephfilesystems.ceph.rook.io
|
||||
spec:
|
||||
group: ceph.rook.io
|
||||
names:
|
||||
kind: CephFilesystem
|
||||
listKind: CephFilesystemList
|
||||
plural: cephfilesystems
|
||||
singular: cephfilesystem
|
||||
scope: Namespaced
|
||||
version: v1
|
||||
additionalPrinterColumns:
|
||||
- name: MdsCount
|
||||
type: string
|
||||
description: Number of MDSs
|
||||
JSONPath: .spec.metadataServer.activeCount
|
||||
- name: Age
|
||||
type: date
|
||||
JSONPath: .metadata.creationTimestamp
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: cephnfses.ceph.rook.io
|
||||
spec:
|
||||
group: ceph.rook.io
|
||||
names:
|
||||
kind: CephNFS
|
||||
listKind: CephNFSList
|
||||
plural: cephnfses
|
||||
singular: cephnfs
|
||||
shortNames:
|
||||
- nfs
|
||||
scope: Namespaced
|
||||
version: v1
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: cephobjectstores.ceph.rook.io
|
||||
spec:
|
||||
group: ceph.rook.io
|
||||
names:
|
||||
kind: CephObjectStore
|
||||
listKind: CephObjectStoreList
|
||||
plural: cephobjectstores
|
||||
singular: cephobjectstore
|
||||
scope: Namespaced
|
||||
version: v1
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: cephobjectstoreusers.ceph.rook.io
|
||||
spec:
|
||||
group: ceph.rook.io
|
||||
names:
|
||||
kind: CephObjectStoreUser
|
||||
listKind: CephObjectStoreUserList
|
||||
plural: cephobjectstoreusers
|
||||
singular: cephobjectstoreuser
|
||||
scope: Namespaced
|
||||
version: v1
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: cephblockpools.ceph.rook.io
|
||||
spec:
|
||||
group: ceph.rook.io
|
||||
names:
|
||||
kind: CephBlockPool
|
||||
listKind: CephBlockPoolList
|
||||
plural: cephblockpools
|
||||
singular: cephblockpool
|
||||
scope: Namespaced
|
||||
version: v1
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: volumes.rook.io
|
||||
spec:
|
||||
group: rook.io
|
||||
names:
|
||||
kind: Volume
|
||||
listKind: VolumeList
|
||||
plural: volumes
|
||||
singular: volume
|
||||
shortNames:
|
||||
- rv
|
||||
scope: Namespaced
|
||||
version: v1alpha2
|
||||
---
|
||||
# The cluster role for managing all the cluster-specific resources in a namespace
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-ceph-cluster-mgmt
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- secrets
|
||||
- pods
|
||||
- pods/log
|
||||
- services
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- extensions
|
||||
resources:
|
||||
- deployments
|
||||
- daemonsets
|
||||
- replicasets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
---
|
||||
# The role for the operator to manage resources in the system namespace
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- configmaps
|
||||
- services
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- extensions
|
||||
resources:
|
||||
- daemonsets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- apps
|
||||
resources:
|
||||
- daemonsets
|
||||
- statefulsets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
|
||||
---
|
||||
# The cluster role for managing the Rook CRDs
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-ceph-global
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
# Pod access is needed for fencing
|
||||
- pods
|
||||
# Node access is needed for determining nodes where mons should run
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- events
|
||||
# PVs and PVCs are managed by the Rook provisioner
|
||||
- persistentvolumes
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- storage.k8s.io
|
||||
resources:
|
||||
- storageclasses
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- batch
|
||||
resources:
|
||||
- jobs
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- ceph.rook.io
|
||||
resources:
|
||||
- "*"
|
||||
verbs:
|
||||
- "*"
|
||||
- apiGroups:
|
||||
- rook.io
|
||||
resources:
|
||||
- "*"
|
||||
verbs:
|
||||
- "*"
|
||||
---
|
||||
# Aspects of ceph-mgr that require cluster-wide access
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-mgr-cluster
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- configmaps
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
---
|
||||
# The rook system service account used by the operator, agent, and discovery pods
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
---
|
||||
# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-system
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
---
|
||||
# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-global
|
||||
namespace: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-ceph-global
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
---
|
||||
# The deployment for the rook operator
|
||||
apiVersion: apps/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
namespace: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: rook-ceph-operator
|
||||
spec:
|
||||
serviceAccountName: rook-ceph-system
|
||||
containers:
|
||||
- name: rook-ceph-operator
|
||||
image: rook/ceph:master
|
||||
args: ["ceph", "operator"]
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/rook
|
||||
name: rook-config
|
||||
- mountPath: /etc/ceph
|
||||
name: default-config-dir
|
||||
- mountPath: /etc/ceph-csi/rbd
|
||||
name: csi-rbd-config
|
||||
- mountPath: /etc/ceph-csi/cephfs
|
||||
name: csi-cephfs-config
|
||||
env:
|
||||
# To disable RBAC, uncomment the following:
|
||||
# - name: RBAC_ENABLED
|
||||
# value: "false"
|
||||
# Rook Agent toleration. Will tolerate all taints with all keys.
|
||||
# Choose between NoSchedule, PreferNoSchedule and NoExecute:
|
||||
# - name: AGENT_TOLERATION
|
||||
# value: "NoSchedule"
|
||||
# (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate
|
||||
# - name: AGENT_TOLERATION_KEY
|
||||
# value: "<KeyOfTheTaintToTolerate>"
|
||||
# (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`.
|
||||
# `Any` uses Ceph admin credentials by default/fallback.
|
||||
# For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and
|
||||
# set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name.
|
||||
# to the namespace in which the `mountSecret` Kubernetes secret namespace.
|
||||
# - name: AGENT_MOUNT_SECURITY_MODE
|
||||
# value: "Any"
|
||||
# Set the path where the Rook agent can find the flex volumes
|
||||
# - name: FLEXVOLUME_DIR_PATH
|
||||
# value: "<PathToFlexVolumes>"
|
||||
# Set the path where kernel modules can be found
|
||||
# - name: LIB_MODULES_DIR_PATH
|
||||
# value: "<PathToLibModules>"
|
||||
# Mount any extra directories into the agent container
|
||||
# - name: AGENT_MOUNTS
|
||||
# value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2"
|
||||
# Rook Discover toleration. Will tolerate all taints with all keys.
|
||||
# Choose between NoSchedule, PreferNoSchedule and NoExecute:
|
||||
# - name: DISCOVER_TOLERATION
|
||||
# value: "NoSchedule"
|
||||
# (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate
|
||||
# - name: DISCOVER_TOLERATION_KEY
|
||||
# value: "<KeyOfTheTaintToTolerate>"
|
||||
# Allow rook to create multiple file systems. Note: This is considered
|
||||
# an experimental feature in Ceph as described at
|
||||
# http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster
|
||||
# which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
|
||||
- name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
|
||||
value: "false"
|
||||
# The logging level for the operator: INFO | DEBUG
|
||||
- name: ROOK_LOG_LEVEL
|
||||
value: "INFO"
|
||||
# The interval to check if every mon is in the quorum.
|
||||
- name: ROOK_MON_HEALTHCHECK_INTERVAL
|
||||
value: "45s"
|
||||
# The duration to wait before trying to failover or remove/replace the
|
||||
# current mon with a new mon (useful for compensating flapping network).
|
||||
- name: ROOK_MON_OUT_TIMEOUT
|
||||
value: "300s"
|
||||
# The duration between discovering devices in the rook-discover daemonset.
|
||||
- name: ROOK_DISCOVER_DEVICES_INTERVAL
|
||||
value: "60m"
|
||||
# Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods.
|
||||
# This is necessary to workaround the anyuid issues when running on OpenShift.
|
||||
# For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
|
||||
- name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
|
||||
value: "false"
|
||||
# In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins).
|
||||
# Disable it here if you have similiar issues.
|
||||
# For more details see https://github.com/rook/rook/issues/2417
|
||||
- name: ROOK_ENABLE_SELINUX_RELABELING
|
||||
value: "true"
|
||||
# CSI enablement
|
||||
- name: ROOK_CSI_ENABLE_CEPHFS
|
||||
value: "true"
|
||||
- name: ROOK_CSI_CEPHFS_IMAGE
|
||||
value: "quay.io/cephcsi/cephfsplugin:v1.0.0"
|
||||
- name: ROOK_CSI_ENABLE_RBD
|
||||
value: "true"
|
||||
- name: ROOK_CSI_RBD_IMAGE
|
||||
value: "quay.io/cephcsi/rbdplugin:v1.0.0"
|
||||
- name: ROOK_CSI_REGISTRAR_IMAGE
|
||||
value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2"
|
||||
- name: ROOK_CSI_PROVISIONER_IMAGE
|
||||
value: "quay.io/k8scsi/csi-provisioner:v1.0.1"
|
||||
- name: ROOK_CSI_SNAPSHOTTER_IMAGE
|
||||
value: "quay.io/k8scsi/csi-snapshotter:v1.0.1"
|
||||
- name: ROOK_CSI_ATTACHER_IMAGE
|
||||
value: "quay.io/k8scsi/csi-attacher:v1.0.1"
|
||||
# The name of the node to pass with the downward API
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
# The pod name to pass with the downward API
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
# The pod namespace to pass with the downward API
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
volumes:
|
||||
- name: rook-config
|
||||
emptyDir: {}
|
||||
- name: default-config-dir
|
||||
emptyDir: {}
|
||||
- name: csi-rbd-config
|
||||
configMap:
|
||||
name: csi-rbd-config
|
||||
- name: csi-cephfs-config
|
||||
configMap:
|
||||
name: csi-cephfs-config
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
|
||||
"github.com/rook/rook/pkg/operator/ceph"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/flags"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -42,6 +43,25 @@ https://github.com/rook/rook`,
|
||||
func init() {
|
||||
operatorCmd.Flags().DurationVar(&mon.HealthCheckInterval, "mon-healthcheck-interval", mon.HealthCheckInterval, "mon health check interval (duration)")
|
||||
operatorCmd.Flags().DurationVar(&mon.MonOutTimeout, "mon-out-timeout", mon.MonOutTimeout, "mon out timeout (duration)")
|
||||
|
||||
operatorCmd.Flags().BoolVar(&csi.EnableRBD, "csi-enable-rbd", false, "whether enable ceph-csi rbd driver")
|
||||
operatorCmd.Flags().BoolVar(&csi.EnableCephFS, "csi-enable-cephfs", false, "whether enable ceph-csi cephfs driver")
|
||||
// csi images
|
||||
operatorCmd.Flags().StringVar(&csi.CSIParam.RBDPluginImage, "csi-rbd-image", csi.DefaultRBDPluginImage, "ceph-csi rbd plugin image")
|
||||
operatorCmd.Flags().StringVar(&csi.CSIParam.CephFSPluginImage, "csi-cephfs-image", csi.DefaultCephFSPluginImage, "ceph-csi cephfs plugin image")
|
||||
operatorCmd.Flags().StringVar(&csi.CSIParam.RegistrarImage, "csi-registrar-image", csi.DefaultRegistrarImage, "csi registrar image")
|
||||
operatorCmd.Flags().StringVar(&csi.CSIParam.ProvisionerImage, "csi-provisioner-image", csi.DefaultProvisionerImage, "csi provisioner image")
|
||||
operatorCmd.Flags().StringVar(&csi.CSIParam.AttacherImage, "csi-attacher-image", csi.DefaultAttacherImage, "csi attacher image")
|
||||
operatorCmd.Flags().StringVar(&csi.CSIParam.SnapshotterImage, "csi-snapshotter-image", csi.DefaultSnapshotterImage, "csi snapshotter image")
|
||||
|
||||
// csi deployment templates
|
||||
operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template")
|
||||
operatorCmd.Flags().StringVar(&csi.RBDProvisionerTemplatePath, "csi-rbd-provisioner-template-path", csi.DefaultRBDProvisionerTemplatePath, "path to ceph-csi rbd provisioner template")
|
||||
operatorCmd.Flags().StringVar(&csi.RBDAttacherTemplatePath, "csi-rbd-attacher-template-path", csi.DefaultRBDAttacherTemplatePath, "path to ceph-csi rbd attacher template")
|
||||
|
||||
operatorCmd.Flags().StringVar(&csi.CephFSPluginTemplatePath, "csi-cephfs-plugin-template-path", csi.DefaultCephFSPluginTemplatePath, "path to ceph-csi cephfs plugin template")
|
||||
operatorCmd.Flags().StringVar(&csi.CephFSProvisionerTemplatePath, "csi-cephfs-provisioner-template-path", csi.DefaultCephFSProvisionerTemplatePath, "path to ceph-csi cephfs provisioner template")
|
||||
|
||||
flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
flags.SetLoggingFlags(operatorCmd.Flags())
|
||||
operatorCmd.RunE = startOperator
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
../../../../cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml
|
||||
@@ -0,0 +1 @@
|
||||
../../../../cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
|
||||
@@ -0,0 +1 @@
|
||||
../../../../cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-attacher.yaml
|
||||
@@ -0,0 +1 @@
|
||||
../../../../cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml
|
||||
@@ -0,0 +1 @@
|
||||
../../../../cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml
|
||||
@@ -0,0 +1,198 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
|
||||
apps "k8s.io/api/apps/v1beta2"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
type Param struct {
|
||||
Namespace string
|
||||
|
||||
RBDPluginImage string
|
||||
CephFSPluginImage string
|
||||
RegistrarImage string
|
||||
ProvisionerImage string
|
||||
AttacherImage string
|
||||
SnapshotterImage string
|
||||
}
|
||||
|
||||
var (
|
||||
CSIParam Param
|
||||
|
||||
EnableRBD = true
|
||||
EnableCephFS = true
|
||||
|
||||
// template paths
|
||||
RBDPluginTemplatePath string
|
||||
RBDProvisionerTemplatePath string
|
||||
RBDAttacherTemplatePath string
|
||||
|
||||
CephFSPluginTemplatePath string
|
||||
CephFSProvisionerTemplatePath string
|
||||
)
|
||||
|
||||
const (
|
||||
KubeMinMajor = "1"
|
||||
KubeMinMinor = "13"
|
||||
|
||||
// image names
|
||||
DefaultRBDPluginImage = "quay.io/cephcsi/rbdplugin:v1.0.0"
|
||||
DefaultCephFSPluginImage = "quay.io/cephcsi/cephfsplugin:v1.0.0"
|
||||
DefaultRegistrarImage = "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2"
|
||||
DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.0.1"
|
||||
DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v1.0.1"
|
||||
DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.0.1"
|
||||
|
||||
// template
|
||||
DefaultRBDPluginTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml"
|
||||
DefaultRBDProvisionerTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml"
|
||||
DefaultRBDAttacherTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-attacher.yaml"
|
||||
DefaultCephFSPluginTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml"
|
||||
DefaultCephFSProvisionerTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner.yaml"
|
||||
|
||||
ExitOnError = false // don't exit if CSI fails to deploy. Switch to true when flexdriver is disabled
|
||||
)
|
||||
|
||||
func CSIEnabled() bool {
|
||||
if EnableRBD || EnableCephFS {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func SetCSINamespace(namespace string) {
|
||||
CSIParam.Namespace = namespace
|
||||
}
|
||||
|
||||
func ValidateCSIParam() error {
|
||||
if EnableRBD {
|
||||
if len(CSIParam.RBDPluginImage) == 0 {
|
||||
return fmt.Errorf("missing csi rbd plugin image")
|
||||
}
|
||||
if len(CSIParam.RegistrarImage) == 0 {
|
||||
return fmt.Errorf("missing csi registrar image")
|
||||
}
|
||||
if len(CSIParam.ProvisionerImage) == 0 {
|
||||
return fmt.Errorf("missing csi provisioner image")
|
||||
}
|
||||
if len(CSIParam.AttacherImage) == 0 {
|
||||
return fmt.Errorf("missing csi attacher image")
|
||||
}
|
||||
if len(RBDPluginTemplatePath) == 0 {
|
||||
return fmt.Errorf("missing rbd plugin template path")
|
||||
}
|
||||
if len(RBDProvisionerTemplatePath) == 0 {
|
||||
return fmt.Errorf("missing rbd provisioner template path")
|
||||
}
|
||||
if len(RBDAttacherTemplatePath) == 0 {
|
||||
return fmt.Errorf("missing rbd attacher template path")
|
||||
}
|
||||
}
|
||||
|
||||
if EnableCephFS {
|
||||
if len(CSIParam.CephFSPluginImage) == 0 {
|
||||
return fmt.Errorf("missing csi cephfs plugin image")
|
||||
}
|
||||
if len(CSIParam.RegistrarImage) == 0 {
|
||||
return fmt.Errorf("missing csi registrar image")
|
||||
}
|
||||
if len(CSIParam.ProvisionerImage) == 0 {
|
||||
return fmt.Errorf("missing csi provisioner image")
|
||||
}
|
||||
if len(CephFSPluginTemplatePath) == 0 {
|
||||
return fmt.Errorf("missing cephfs plugin template path")
|
||||
}
|
||||
if len(CephFSProvisionerTemplatePath) == 0 {
|
||||
return fmt.Errorf("missing ceph provisioner template path")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func StartCSIDrivers(namespace string, clientset kubernetes.Interface) error {
|
||||
var (
|
||||
err error
|
||||
rbdPlugin, cephfsPlugin *apps.DaemonSet
|
||||
rbdProvisioner, rbdAttacher, cephfsProvisioner *apps.StatefulSet
|
||||
)
|
||||
|
||||
if EnableRBD {
|
||||
rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load rbd plugin template: %v", err)
|
||||
}
|
||||
rbdProvisioner, err = templateToStatefulSet("rbd-provisioner", RBDProvisionerTemplatePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load rbd provisioner template: %v", err)
|
||||
}
|
||||
rbdAttacher, err = templateToStatefulSet("rbd-attacher", RBDAttacherTemplatePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load rbd attacher template: %v", err)
|
||||
}
|
||||
}
|
||||
if EnableCephFS {
|
||||
cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load CephFS plugin template: %v", err)
|
||||
}
|
||||
cephfsProvisioner, err = templateToStatefulSet("cephfs-provisioner", CephFSProvisionerTemplatePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load CephFS provisioner template: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if rbdPlugin != nil {
|
||||
err = k8sutil.CreateDaemonSet("csi rbd plugin", namespace, clientset, rbdPlugin)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start rbdplugin daemonset: %v\n%v", err, rbdPlugin)
|
||||
}
|
||||
}
|
||||
if rbdProvisioner != nil {
|
||||
_, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisioner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisioner)
|
||||
}
|
||||
|
||||
}
|
||||
if rbdAttacher != nil {
|
||||
_, err = k8sutil.CreateStatefulSet("csi rbd attacher", namespace, "csi-rbdplugin-attacher", clientset, rbdAttacher)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start rbd attacher statefulset: %v\n%v", err, rbdAttacher)
|
||||
}
|
||||
}
|
||||
|
||||
if cephfsPlugin != nil {
|
||||
err = k8sutil.CreateDaemonSet("csi cephfs plugin", namespace, clientset, cephfsPlugin)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start cephfs plugin daemonset: %v\n%v", err, cephfsPlugin)
|
||||
}
|
||||
}
|
||||
if cephfsProvisioner != nil {
|
||||
_, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisioner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisioner)
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestStartCSI(t *testing.T) {
|
||||
RBDPluginTemplatePath = "csi-rbdplugin.yaml"
|
||||
RBDProvisionerTemplatePath = "csi-rbdplugin-provisioner.yaml"
|
||||
RBDAttacherTemplatePath = "csi-rbdplugin-attacher.yaml"
|
||||
|
||||
CephFSPluginTemplatePath = "csi-cephfsplugin.yaml"
|
||||
CephFSProvisionerTemplatePath = "csi-cephfsplugin-provisioner.yaml"
|
||||
|
||||
CSIParam = Param{
|
||||
RBDPluginImage: "image",
|
||||
CephFSPluginImage: "image",
|
||||
RegistrarImage: "image",
|
||||
ProvisionerImage: "image",
|
||||
AttacherImage: "image",
|
||||
SnapshotterImage: "image",
|
||||
}
|
||||
clientset := test.New(3)
|
||||
err := StartCSIDrivers("ns", clientset)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"text/template"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
|
||||
apps "k8s.io/api/apps/v1beta2"
|
||||
)
|
||||
|
||||
func loadTemplate(name, templatePath string) (string, error) {
|
||||
b, err := ioutil.ReadFile(templatePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data := string(b)
|
||||
var writer bytes.Buffer
|
||||
t := template.New(name)
|
||||
err = template.Must(t.Parse(data)).Execute(&writer, CSIParam)
|
||||
return writer.String(), err
|
||||
}
|
||||
|
||||
func templateToStatefulSet(name, templatePath string) (*apps.StatefulSet, error) {
|
||||
var ss apps.StatefulSet
|
||||
t, err := loadTemplate(name, templatePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal([]byte(t), &ss)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ss, nil
|
||||
}
|
||||
|
||||
func templateToDaemonSet(name, templatePath string) (*apps.DaemonSet, error) {
|
||||
var ds apps.DaemonSet
|
||||
t, err := loadTemplate(name, templatePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal([]byte(t), &ds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ds, nil
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csi
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var (
|
||||
testDSTemplate = []byte(`
|
||||
kind: DaemonSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: test-label
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: test-label
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: test-label
|
||||
spec:
|
||||
serviceAccount: test-sa
|
||||
containers:
|
||||
- name: registrar
|
||||
image: {{ .RegistrarImage }}
|
||||
- name: rbdplugin
|
||||
image: {{ .RBDPluginImage }}
|
||||
- name: cephfsplugin
|
||||
image: {{ .CephFSPluginImage }}
|
||||
`)
|
||||
testSSTemplate = []byte(`
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: test-label
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: test-label
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: test-label
|
||||
spec:
|
||||
serviceAccount: test-sa
|
||||
containers:
|
||||
- name: csi-rbdplugin-attacher
|
||||
image: {{ .AttacherImage }}
|
||||
- name: provisioner
|
||||
image: {{ .ProvisionerImage }}
|
||||
- name: rbdplugin
|
||||
image: {{ .RBDPluginImage }}
|
||||
- name: cephfsplugin
|
||||
image: {{ .CephFSPluginImage }}
|
||||
`)
|
||||
)
|
||||
|
||||
func TestDaemonSetTemplate(t *testing.T) {
|
||||
tmp, err := ioutil.TempFile("", "yaml")
|
||||
assert.Nil(t, err)
|
||||
|
||||
defer os.Remove(tmp.Name())
|
||||
|
||||
_, err = tmp.Write(testDSTemplate)
|
||||
assert.Nil(t, err)
|
||||
err = tmp.Close()
|
||||
assert.Nil(t, err)
|
||||
|
||||
_, err = templateToDaemonSet("test-ds", tmp.Name())
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestStatefulSetTemplate(t *testing.T) {
|
||||
tmp, err := ioutil.TempFile("", "yaml")
|
||||
assert.Nil(t, err)
|
||||
|
||||
defer os.Remove(tmp.Name())
|
||||
|
||||
_, err = tmp.Write(testSSTemplate)
|
||||
assert.Nil(t, err)
|
||||
err = tmp.Close()
|
||||
assert.Nil(t, err)
|
||||
|
||||
_, err = templateToStatefulSet("test-ss", tmp.Name())
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
|
||||
"github.com/rook/rook/pkg/operator/ceph/agent"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster"
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
"github.com/rook/rook/pkg/operator/ceph/file"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object/user"
|
||||
@@ -100,15 +101,35 @@ func (o *Operator) Run() error {
|
||||
return fmt.Errorf("Error starting device discovery daemonset: %v", err)
|
||||
}
|
||||
|
||||
signalChan := make(chan os.Signal, 1)
|
||||
stopChan := make(chan struct{})
|
||||
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
serverVersion, err := o.context.Clientset.Discovery().ServerVersion()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error getting server version: %v", err)
|
||||
}
|
||||
|
||||
if serverVersion.Major >= csi.KubeMinMajor && serverVersion.Minor >= csi.KubeMinMinor && csi.CSIEnabled() {
|
||||
logger.Infof("Ceph CSI driver is enabled, validate csi param")
|
||||
if err = csi.ValidateCSIParam(); err != nil {
|
||||
logger.Warningf("invalid csi params: %v", err)
|
||||
if csi.ExitOnError {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
csi.SetCSINamespace(namespace)
|
||||
if err = csi.StartCSIDrivers(namespace, o.context.Clientset); err != nil {
|
||||
logger.Warningf("failed to start Ceph csi drivers: %v", err)
|
||||
if csi.ExitOnError {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
logger.Infof("successfully started Ceph csi drivers")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
signalChan := make(chan os.Signal, 1)
|
||||
stopChan := make(chan struct{})
|
||||
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
// Run volume provisioner for each of the supported configurations
|
||||
for name, vendor := range provisionerConfigs {
|
||||
volumeProvisioner := provisioner.New(o.context, vendor)
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package k8sutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
apps "k8s.io/api/apps/v1beta2"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
// make a headless svc for statefulset
|
||||
func makeHeadlessSvc(name, namespace string, label map[string]string, clientset kubernetes.Interface) (*corev1.Service, error) {
|
||||
svc := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Labels: label,
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Selector: label,
|
||||
Ports: []corev1.ServicePort{{Name: "dummy", Port: 1234}},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := clientset.CoreV1().Services(namespace).Create(svc)
|
||||
if err != nil && !k8serrors.IsAlreadyExists(err) {
|
||||
return nil, fmt.Errorf("failed to create %s headless service. %+v", name, err)
|
||||
}
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// create a apps.daemonset
|
||||
func CreateDaemonSet(name, namespace string, clientset kubernetes.Interface, ds *apps.DaemonSet) error {
|
||||
_, err := clientset.AppsV1beta2().DaemonSets(namespace).Create(ds)
|
||||
if err != nil {
|
||||
if k8serrors.IsAlreadyExists(err) {
|
||||
_, err = clientset.AppsV1beta2().DaemonSets(namespace).Update(ds)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start %s daemonset: %v\n%v", name, err, ds)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// create a apps.statefulset and a headless svc
|
||||
func CreateStatefulSet(name, namespace, appName string, clientset kubernetes.Interface, ss *apps.StatefulSet) (*corev1.Service, error) {
|
||||
label := ss.GetLabels()
|
||||
svc, err := makeHeadlessSvc(appName, namespace, label, clientset)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to start %s service: %v\n%v", name, err, ss)
|
||||
}
|
||||
|
||||
_, err = clientset.AppsV1beta2().StatefulSets(namespace).Create(ss)
|
||||
if err != nil {
|
||||
if k8serrors.IsAlreadyExists(err) {
|
||||
_, err = clientset.AppsV1beta2().StatefulSets(namespace).Update(ss)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to start %s statefulset: %v\n%v", name, err, ss)
|
||||
}
|
||||
}
|
||||
return svc, err
|
||||
}
|
||||
@@ -0,0 +1,467 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package installer
|
||||
|
||||
const (
|
||||
rbdAttacherTemplate = `
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-rbdplugin-attacher
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
serviceName: csi-rbdplugin-attacher
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-rbdplugin-attacher
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-rbdplugin-attacher
|
||||
spec:
|
||||
serviceAccount: rook-csi-rbd-attacher-sa
|
||||
containers:
|
||||
- name: csi-rbdplugin-attacher
|
||||
image: {{ .AttacherImage }}
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
type: DirectoryOrCreate
|
||||
`
|
||||
rbdProvisionerTemplate = `
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-rbdplugin-provisioner
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
serviceName: "csi-rbdplugin-provisioner"
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-rbdplugin-provisioner
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-rbdplugin-provisioner
|
||||
spec:
|
||||
serviceAccount: rook-csi-rbd-provisioner-sa
|
||||
containers:
|
||||
- name: csi-provisioner
|
||||
image: {{ .ProvisionerImage }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
- name: csi-snapshotter
|
||||
image: {{ .SnapshotterImage }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--connection-timeout=15s"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
|
||||
imagePullPolicy: Always
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
- name: csi-rbdplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
image: {{ .RBDPluginImage }}
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=csi-rbdplugin"
|
||||
- "--containerized=true"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /rootfs
|
||||
name: host-rootfs
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- mountPath: /lib/modules
|
||||
name: lib-modules
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: host-rootfs
|
||||
hostPath:
|
||||
path: /
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
type: DirectoryOrCreate
|
||||
`
|
||||
rbdPluginTemplate = `
|
||||
kind: DaemonSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-rbdplugin
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-rbdplugin
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-rbdplugin
|
||||
spec:
|
||||
serviceAccount: rook-csi-rbd-plugin-sa
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
|
||||
# resolved through k8s service, set dns policy to cluster first
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
containers:
|
||||
- name: driver-registrar
|
||||
image: {{ .RegistrarImage }}
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=/csi/csi.sock"
|
||||
- "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock"
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"]
|
||||
env:
|
||||
- name: KUBE_NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /csi
|
||||
- name: registration-dir
|
||||
mountPath: /registration
|
||||
- name: csi-rbdplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: {{ .RBDPluginImage }}
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=csi-rbdplugin"
|
||||
- "--containerized=true"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin
|
||||
- name: pods-mount-dir
|
||||
mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: plugin-mount-dir
|
||||
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
|
||||
mountPropagation: "Bidirectional"
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /rootfs
|
||||
name: host-rootfs
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- mountPath: /lib/modules
|
||||
name: lib-modules
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: plugin-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-rbdplugin
|
||||
type: DirectoryOrCreate
|
||||
- name: plugin-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
|
||||
type: DirectoryOrCreate
|
||||
- name: registration-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins_registry/
|
||||
type: Directory
|
||||
- name: pods-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: host-rootfs
|
||||
hostPath:
|
||||
path: /
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
`
|
||||
cephfsProvisionerTemplate = `
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-cephfsplugin-provisioner
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
serviceName: "csi-cephfsplugin-provisioner"
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-cephfsplugin-provisioner
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-cephfsplugin-provisioner
|
||||
spec:
|
||||
serviceAccount: rook-csi-cephfs-provisioner-sa
|
||||
containers:
|
||||
- name: csi-provisioner
|
||||
image: {{ .ProvisionerImage }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
|
||||
- name: csi-cephfsplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
image: {{ .CephFSPluginImage }}
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=csi-cephfsplugin"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
|
||||
- name: host-sys
|
||||
mountPath: /sys
|
||||
- name: lib-modules
|
||||
mountPath: /lib/modules
|
||||
readOnly: true
|
||||
- name: host-dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-cephfsplugin
|
||||
type: DirectoryOrCreate
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
`
|
||||
cephfsPluginTemplate = `
|
||||
kind: DaemonSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: csi-cephfsplugin
|
||||
namespace: {{ .Namespace }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-cephfsplugin
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-cephfsplugin
|
||||
spec:
|
||||
serviceAccount: rook-csi-cephfs-plugin-sa
|
||||
hostNetwork: true
|
||||
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
|
||||
# resolved through k8s service, set dns policy to cluster first
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
containers:
|
||||
- name: driver-registrar
|
||||
image: {{ .RegistrarImage }}
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=/csi/csi.sock"
|
||||
- "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock"
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: ["/bin/sh", "-c", "rm -rf /registration/csi-cephfsplugin /registration/csi-cephfsplugin-reg.sock"]
|
||||
env:
|
||||
- name: KUBE_NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /csi
|
||||
- name: registration-dir
|
||||
mountPath: /registration
|
||||
- name: csi-cephfsplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: {{ .CephFSPluginImage }}
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=csi-cephfsplugin"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin
|
||||
- name: csi-plugins-dir
|
||||
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: pods-mount-dir
|
||||
mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: host-sys
|
||||
mountPath: /sys
|
||||
- name: lib-modules
|
||||
mountPath: /lib/modules
|
||||
readOnly: true
|
||||
- name: host-dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: plugin-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/csi-cephfsplugin/
|
||||
type: DirectoryOrCreate
|
||||
- name: csi-plugins-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/kubernetes.io/csi
|
||||
type: DirectoryOrCreate
|
||||
- name: registration-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins_registry/
|
||||
type: Directory
|
||||
- name: pods-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
`
|
||||
)
|
||||
@@ -394,6 +394,21 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-global", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().Roles(systemNamespace).Delete("rook-ceph-system", nil)
|
||||
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-attacher-role", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-external-attacher-runner", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-nodeplugin", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-csi-nodeplugin", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rbd-csi-provisioner-role", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rbd-external-provisioner-runner", nil)
|
||||
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("cephfs-csi-nodeplugin", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-csi-nodeplugin", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("cephfs-csi-provisioner-role", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("cephfs-external-provisioner-runner", nil)
|
||||
|
||||
h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-rbd-config", nil)
|
||||
h.k8shelper.Clientset.CoreV1().ConfigMaps(systemNamespace).Delete("csi-cephfs-config", nil)
|
||||
|
||||
logger.Infof("done removing the operator from namespace %s", systemNamespace)
|
||||
logger.Infof("removing host data dir %s", h.hostPathToDelete)
|
||||
// removing data dir if exists
|
||||
|
||||
@@ -246,6 +246,7 @@ rules:
|
||||
resources:
|
||||
- pods
|
||||
- configmaps
|
||||
- services
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
@@ -265,6 +266,19 @@ rules:
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- apps
|
||||
resources:
|
||||
- daemonsets
|
||||
- statefulsets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
@@ -444,6 +458,250 @@ subjects:
|
||||
name: rook-ceph-system
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: csi-rbd-config
|
||||
namespace: ` + namespace + `
|
||||
data:
|
||||
csi-rbdplugin-attacher.yaml: |-` + rbdAttacherTemplate + `
|
||||
csi-rbdplugin-provisioner.yaml: |-` + rbdProvisionerTemplate + `
|
||||
csi-rbdplugin.yaml: |-` + rbdPluginTemplate + `
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: csi-cephfs-config
|
||||
namespace: ` + namespace + `
|
||||
data:
|
||||
csi-cephfsplugin-provisioner.yaml: |` + cephfsProvisionerTemplate + `
|
||||
csi-cephfsplugin.yaml: |` + cephfsPluginTemplate + `
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-rbd-attacher-sa
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-external-attacher-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-csi-attacher-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-rbd-attacher-sa
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: rbd-external-attacher-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-rbd-plugin-sa
|
||||
namespace: ` + namespace + `
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-csi-nodeplugin
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-csi-nodeplugin
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-rbd-plugin-sa
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: rbd-csi-nodeplugin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-rbd-provisioner-sa
|
||||
namespace: ` + namespace + `
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-external-provisioner-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["endpoints"]
|
||||
verbs: ["get", "create", "update"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "create", "delete"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "delete"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["apiextensions.k8s.io"]
|
||||
resources: ["customresourcedefinitions"]
|
||||
verbs: ["create"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-csi-provisioner-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-rbd-provisioner-sa
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: rbd-external-provisioner-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-cephfs-plugin-sa
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: cephfs-csi-nodeplugin
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: cephfs-csi-nodeplugin
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-cephfs-plugin-sa
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: cephfs-csi-nodeplugin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-csi-cephfs-provisioner-sa
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: cephfs-external-provisioner-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "create", "delete"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: cephfs-csi-provisioner-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-csi-cephfs-provisioner-sa
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: cephfs-external-provisioner-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: apps/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -482,7 +740,37 @@ spec:
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace`
|
||||
fieldPath: metadata.namespace
|
||||
# CSI enablement
|
||||
- name: ROOK_CSI_ENABLE_CEPHFS
|
||||
value: "true"
|
||||
- name: ROOK_CSI_CEPHFS_IMAGE
|
||||
value: "quay.io/cephcsi/cephfsplugin:v1.0.0"
|
||||
- name: ROOK_CSI_ENABLE_RBD
|
||||
value: "true"
|
||||
- name: ROOK_CSI_RBD_IMAGE
|
||||
value: "quay.io/cephcsi/rbdplugin:v1.0.0"
|
||||
- name: ROOK_CSI_REGISTRAR_IMAGE
|
||||
value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2"
|
||||
- name: ROOK_CSI_PROVISIONER_IMAGE
|
||||
value: "quay.io/k8scsi/csi-provisioner:v1.0.1"
|
||||
- name: ROOK_CSI_SNAPSHOTTER_IMAGE
|
||||
value: "quay.io/k8scsi/csi-snapshotter:v1.0.1"
|
||||
- name: ROOK_CSI_ATTACHER_IMAGE
|
||||
value: "quay.io/k8scsi/csi-attacher:v1.0.1"
|
||||
volumeMounts:
|
||||
- mountPath: /etc/ceph-csi/rbd
|
||||
name: csi-rbd-config
|
||||
- mountPath: /etc/ceph-csi/cephfs
|
||||
name: csi-cephfs-config
|
||||
volumes:
|
||||
- name: csi-rbd-config
|
||||
configMap:
|
||||
name: csi-rbd-config
|
||||
- name: csi-cephfs-config
|
||||
configMap:
|
||||
name: csi-cephfs-config
|
||||
`
|
||||
}
|
||||
|
||||
// GetClusterRoles returns rook-cluster manifest
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
monclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
csiSecretName = "ceph-csi-secret"
|
||||
csiSCRBD = "ceph-csi-rbd"
|
||||
csiSCCephFS = "ceph-csi-cephfs"
|
||||
csiPoolRBD = "csi-rbd"
|
||||
csiPoolCephFS = "csi-cephfs"
|
||||
csiTestRBDPodName = "csi-test-rbd"
|
||||
csiTestCephFSPodName = "csi-test-cephfs"
|
||||
)
|
||||
|
||||
func runCephCSIE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
|
||||
if isCSIRunnig(k8sh, namespace) {
|
||||
logger.Info("test Ceph CSI driver")
|
||||
createCephCSISecret(helper, k8sh, s, namespace)
|
||||
createCephPools(helper, s, namespace)
|
||||
createCSIStorageClass(k8sh, s, namespace)
|
||||
createCSIRBDTestPod(k8sh, s, namespace)
|
||||
}
|
||||
}
|
||||
|
||||
func isCSIRunnig(k8sh *utils.K8sHelper, namespace string) bool {
|
||||
return k8sh.IsPodWithLabelPresent("app=csi-rbdplugin", installer.SystemNamespace(namespace))
|
||||
}
|
||||
|
||||
func createCephCSISecret(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
|
||||
commandArgs := []string{"-c", "ceph auth get-key client.admin"}
|
||||
keyResult, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs)
|
||||
logger.Infof("Ceph get-key: %s", keyResult)
|
||||
require.Nil(s.T(), err)
|
||||
commandArgs = []string{"-c", "ceph mon_status"}
|
||||
monResult, err := k8sh.Exec(namespace, "rook-ceph-tools", "bash", commandArgs)
|
||||
logger.Infof("Ceph mon_status: %s", monResult)
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
var mon monclient.MonStatusResponse
|
||||
err = json.Unmarshal([]byte(monResult), &mon)
|
||||
require.Nil(s.T(), err)
|
||||
require.True(s.T(), len(mon.MonMap.Mons) > 0, "no mon found")
|
||||
monStr := strings.Split(mon.MonMap.Mons[0].Address, "/")[0]
|
||||
require.True(s.T(), len(monStr) > 0, "invalid mon addr")
|
||||
|
||||
_, err = k8sh.Clientset.Core().Secrets(namespace).Create(&v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: csiSecretName,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"admin": []byte(keyResult),
|
||||
"adminID": []byte("admin"),
|
||||
"adminKey": []byte(keyResult),
|
||||
"monitors": []byte(monStr),
|
||||
},
|
||||
})
|
||||
require.Nil(s.T(), err)
|
||||
logger.Info("Created Ceph CSI Secret")
|
||||
}
|
||||
|
||||
func createCephPools(helper *clients.TestClient, s suite.Suite, namespace string) {
|
||||
out, err := helper.PoolClient.Create(csiPoolRBD, namespace, 1)
|
||||
logger.Infof("rbd pool create: %+v", out)
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
out, err = helper.PoolClient.Create(csiPoolCephFS, namespace, 1)
|
||||
logger.Infof("cephFS pool create: %+v", out)
|
||||
require.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
func createCSIStorageClass(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
|
||||
rbdSC := `
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ` + csiSCRBD + `
|
||||
provisioner: csi-rbdplugin
|
||||
parameters:
|
||||
monValueFromSecret: "monitors"
|
||||
pool: ` + csiPoolRBD + `
|
||||
csi.storage.k8s.io/provisioner-secret-name: ` + csiSecretName + `
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + `
|
||||
csi.storage.k8s.io/node-publish-secret-name: ` + csiSecretName + `
|
||||
csi.storage.k8s.io/node-publish-secret-namespace: ` + namespace + `
|
||||
adminid: admin
|
||||
userid: admin
|
||||
`
|
||||
cephFSSC := `
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ` + csiSCCephFS + `
|
||||
provisioner: csi-cephFSplugin
|
||||
parameters:
|
||||
monValueFromSecret: "monitors"
|
||||
pool: ` + csiPoolCephFS + `
|
||||
csi.storage.k8s.io/provisioner-secret-name: ` + csiSecretName + `
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: ` + namespace + `
|
||||
csi.storage.k8s.io/node-stage-secret-name: ` + csiSecretName + `
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: ` + namespace + `
|
||||
adminid: admin
|
||||
userid: admin
|
||||
`
|
||||
out, err := k8sh.ResourceOperation("apply", rbdSC)
|
||||
logger.Infof("create rbd storage class: %v", out)
|
||||
require.Nil(s.T(), err)
|
||||
|
||||
out, err = k8sh.ResourceOperation("apply", cephFSSC)
|
||||
logger.Infof("create cephfs storage class: %v", out)
|
||||
require.Nil(s.T(), err)
|
||||
}
|
||||
|
||||
func createCSIRBDTestPod(k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
|
||||
pod := `
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: rbd-pvc-csi
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
storageClassName: ` + csiSCRBD + `
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: ` + csiTestRBDPodName + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
containers:
|
||||
- name: ` + csiTestRBDPodName + `
|
||||
image: busybox
|
||||
command:
|
||||
- sh
|
||||
- "-c"
|
||||
- "touch /test/csi.test && sleep 3600"
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
volumeMounts:
|
||||
- mountPath: /test
|
||||
name: csivol
|
||||
volumes:
|
||||
- name: csivol
|
||||
persistentVolumeClaim:
|
||||
claimName: rbd-pvc-csi
|
||||
readOnly: false
|
||||
restartPolicy: Never
|
||||
`
|
||||
out, err := k8sh.ResourceOperation("create", pod)
|
||||
logger.Infof("create csi rbd test pod: %v", out)
|
||||
require.Nil(s.T(), err)
|
||||
isPodRunning := k8sh.IsPodRunning(csiTestRBDPodName, namespace)
|
||||
if !isPodRunning {
|
||||
k8sh.PrintPodDescribe(namespace, csiTestRBDPodName)
|
||||
k8sh.PrintPodStatus(namespace)
|
||||
} else {
|
||||
// cleanup the pod and pv
|
||||
_, err = k8sh.ResourceOperation("delete", pod)
|
||||
}
|
||||
|
||||
require.True(s.T(), isPodRunning, "csi rbd test pod fails to run")
|
||||
}
|
||||
@@ -81,6 +81,10 @@ func (suite *SmokeSuite) TearDownSuite() {
|
||||
suite.op.Teardown()
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TestBlockCSI_SmokeTest() {
|
||||
runCephCSIE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TestBlockStorage_SmokeTest() {
|
||||
runBlockE2ETest(suite.helper, suite.k8sh, suite.Suite, suite.namespace)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user