diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index f7320f553..6c76a1373 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -66,6 +66,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.enableRbdDriver` | Enable Ceph CSI RBD driver. | `true` | | `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver. | `true` | | `csi.enableCephfsSnapshotter` | Enable Snapshotter in CephFS provisioner pod. | `true` | +| `csi.enableNFSSnapshotter` | Enable Snapshotter in NFS provisioner pod. | `true` | | `csi.enableRBDSnapshotter` | Enable Snapshotter in RBD provisioner pod. | `true` | | `csi.pluginPriorityClassName` | PriorityClassName to be set on csi driver plugin pods. | `` | | `csi.provisionerPriorityClassName` | PriorityClassName to be set on csi driver provisioner pods. | `` | diff --git a/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md b/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md index e03383f07..640d12f56 100644 --- a/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md +++ b/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md @@ -97,3 +97,109 @@ application pod. ### Connecting to an export directly After a PVC is created successfully, the `share` parameter set on the resulting PV contains the `share` path which can be used as the export path when [mounting the export manually](nfs.md#mounting-exports). + +## Taking snapshots of NFS exports + +NFS export PVCs can be snapshotted and later restored to new PVCs. + +### Creating snapshots + +First, create a VolumeSnapshotClass as in the example [here](https://github.com/rook/rook/tree/master/deploy/examples/csi/nfs/snapshotclass.yaml). The `csi.storage.k8s.io/snapshotter-secret-name` parameter should reference the name of the secret created for the cephfsplugin [here](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/snapshotclass.yaml). + +```console +kubectl create -f deploy/examples/csi/nfs/snapshotclass.yaml +``` + +In [snapshot](https://github.com/rook/rook/tree/master/deploy/examples/csi/nfs/snapshot.yaml), +`volumeSnapshotClassName` should be the name of the VolumeSnapshotClass +previously created. The `persistentVolumeClaimName` should be the name of the +PVC which is already created by the NFS CSI driver. + +```console +kubectl create -f deploy/examples/csi/nfs/snapshot.yaml +``` + +### Verifying snapshots + +```console +$ kubectl get volumesnapshotclass +NAME DRIVER DELETIONPOLICY AGE +csi-nfslugin-snapclass rook-ceph.nfs.csi.ceph.com Delete 3h55m +``` + +```console +$ kubectl get volumesnapshot +NAME READYTOUSE SOURCEPVC SOURCESNAPSHOTCONTENT RESTORESIZE SNAPSHOTCLASS SNAPSHOTCONTENT CREATIONTIME AGE +nfs-pvc-snapshot true nfs-pvc 1Gi csi-nfsplugin-snapclass snapcontent-34476204-a14a-4d59-bfbc-2bbba695652c 3h50m 3h51m +``` + +The snapshot will be ready to restore to a new PVC when `READYTOUSE` field of the +volumesnapshot is set to true. + +### Restoring snapshot to a new PVC + +In +[pvc-restore](https://github.com/rook/rook/tree/master/deploy/examples/csi/nfs/pvc-restore.yaml), +`dataSource` name should be the name of the VolumeSnapshot previously +created. The `dataSource` kind should be "VolumeSnapshot". + +Create a new PVC from the snapshot. + +```console +kubectl create -f deploy/examples/csi/nfs/pvc-restore.yaml +``` + +### Verifying restored PVC Creation + +```console +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +nfs-pvc Bound pvc-74734901-577a-11e9-b34f-525400581048 1Gi RWX rook-nfs 55m +nfs-pvc-restore Bound pvc-95308c75-6c93-4928-a551-6b5137192209 1Gi RWX rook-nfs 34s +``` + +### Cleaning up snapshot resource + +To clean your cluster of the resources created by this example, run the following: + +```console +kubectl delete -f deploy/examples/csi/nfs/pvc-restore.yaml +kubectl delete -f deploy/examples/csi/nfs/snapshot.yaml +kubectl delete -f deploy/examples/csi/nfs/snapshotclass.yaml +``` + +## Cloning NFS exports + +### Creating clones + +In +[pvc-clone](https://github.com/rook/rook/tree/master/deploy/examples/csi/nfs/pvc-clone.yaml), +`dataSource` should be the name of the PVC which is already created by NFS +CSI driver. The `dataSource` kind should be "PersistentVolumeClaim" and also storageclass +should be same as the source PVC. + +Create a new PVC Clone from the PVC as in the example [here](https://github.com/rook/rook/tree/master/deploy/examples/csi/nfs/pvc-clone.yaml). + +```console +kubectl create -f deploy/examples/csi/nfs/pvc-clone.yaml +``` + +### Verifying a cloned PVC + +```console +kubectl get pvc +``` + +>``` +>NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +>nfs-pvc Bound pvc-1ea51547-a88b-4ab0-8b4a-812caeaf025d 1Gi RWX rook-nfs 39m +>nfs-pvc-clone Bound pvc-b575bc35-d521-4c41-b4f9-1d733cd28fdf 1Gi RWX rook-nfs 8s +>``` + +### Cleaning up clone resources + +To clean your cluster of the resources created by this example, run the following: + +```console +kubectl delete -f deploy/examples/csi/nfs/pvc-clone.yaml +``` diff --git a/deploy/charts/rook-ceph/templates/clusterrole.yaml b/deploy/charts/rook-ceph/templates/clusterrole.yaml index 4673e582e..753dd84c2 100644 --- a/deploy/charts/rook-ceph/templates/clusterrole.yaml +++ b/deploy/charts/rook-ceph/templates/clusterrole.yaml @@ -395,7 +395,7 @@ metadata: rules: - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "patch"] @@ -417,6 +417,21 @@ rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] --- # TODO: remove this, once https://github.com/rook/rook/issues/10141 # is resolved. diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 8056ace6c..681ecae4d 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -12,6 +12,7 @@ data: ROOK_CSI_ENABLE_RBD: {{ .Values.csi.enableRbdDriver | quote }} ROOK_CSI_ENABLE_CEPHFS: {{ .Values.csi.enableCephfsDriver | quote }} CSI_ENABLE_CEPHFS_SNAPSHOTTER: {{ .Values.csi.enableCephfsSnapshotter | quote }} + CSI_ENABLE_NFS_SNAPSHOTTER: {{ .Values.csi.enableNFSSnapshotter | quote }} CSI_ENABLE_RBD_SNAPSHOTTER: {{ .Values.csi.enableRBDSnapshotter | quote }} CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: {{ .Values.csi.enablePluginSelinuxHostMount | quote }} CSI_ENABLE_ENCRYPTION: {{ .Values.csi.enableCSIEncryption | quote }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index b3c780346..17d5cf209 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -66,6 +66,8 @@ csi: enableCSIHostNetwork: true # set to false to disable deployment of snapshotter container in CephFS provisioner pod. enableCephfsSnapshotter: true + # set to false to disable deployment of snapshotter container in NFS provisioner pod. + enableNFSSnapshotter: true # set to false to disable deployment of snapshotter container in RBD provisioner pod. enableRBDSnapshotter: true # set to false if the selinux is not enabled or unavailable in cluster nodes. diff --git a/deploy/examples/csi/nfs/pvc-clone.yaml b/deploy/examples/csi/nfs/pvc-clone.yaml new file mode 100644 index 000000000..a4830353a --- /dev/null +++ b/deploy/examples/csi/nfs/pvc-clone.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nfs-pvc-clone +spec: + storageClassName: rook-nfs + dataSource: + name: nfs-pvc + kind: PersistentVolumeClaim + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi diff --git a/deploy/examples/csi/nfs/pvc-restore.yaml b/deploy/examples/csi/nfs/pvc-restore.yaml new file mode 100644 index 000000000..8a9d68c1f --- /dev/null +++ b/deploy/examples/csi/nfs/pvc-restore.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nfs-pvc-restore +spec: + storageClassName: rook-nfs + dataSource: + name: nfs-pvc-snapshot + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi diff --git a/deploy/examples/csi/nfs/rbac.yaml b/deploy/examples/csi/nfs/rbac.yaml index 7dc981f60..b8dbef640 100644 --- a/deploy/examples/csi/nfs/rbac.yaml +++ b/deploy/examples/csi/nfs/rbac.yaml @@ -24,7 +24,7 @@ metadata: rules: - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "patch"] @@ -46,6 +46,21 @@ rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] --- # TODO: remove this, once https://github.com/rook/rook/issues/10141 # is resolved. diff --git a/deploy/examples/csi/nfs/snapshot.yaml b/deploy/examples/csi/nfs/snapshot.yaml new file mode 100644 index 000000000..bf1bd973a --- /dev/null +++ b/deploy/examples/csi/nfs/snapshot.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: nfs-pvc-snapshot +spec: + volumeSnapshotClassName: csi-nfsplugin-snapclass + source: + persistentVolumeClaimName: nfs-pvc diff --git a/deploy/examples/csi/nfs/snapshotclass.yaml b/deploy/examples/csi/nfs/snapshotclass.yaml new file mode 100644 index 000000000..b11cefd35 --- /dev/null +++ b/deploy/examples/csi/nfs/snapshotclass.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: csi-nfsplugin-snapclass +driver: rook-ceph.nfs.csi.ceph.com # driver:namespace:operator +parameters: + # Specify a string that identifies your cluster. Ceph CSI supports any + # unique string. When Ceph CSI is deployed by Rook use the Rook namespace, + # for example "rook-ceph". + clusterID: rook-ceph # namespace:cluster + csi.storage.k8s.io/snapshotter-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/snapshotter-secret-namespace: rook-ceph # namespace:cluster +deletionPolicy: Delete diff --git a/deploy/examples/csi/nfs/storageclass.yaml b/deploy/examples/csi/nfs/storageclass.yaml index 57818282c..85e60f754 100644 --- a/deploy/examples/csi/nfs/storageclass.yaml +++ b/deploy/examples/csi/nfs/storageclass.yaml @@ -42,7 +42,7 @@ parameters: csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster reclaimPolicy: Delete -allowVolumeExpansion: false +allowVolumeExpansion: true mountOptions: # uncomment the following line for debugging #- debug diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 490bceda7..1cf3c5884 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -155,6 +155,9 @@ data: # set to false to disable deployment of snapshotter container in CephFS provisioner pod. CSI_ENABLE_CEPHFS_SNAPSHOTTER: "true" + # set to false to disable deployment of snapshotter container in NFS provisioner pod. + CSI_ENABLE_NFS_SNAPSHOTTER: "true" + # set to false to disable deployment of snapshotter container in RBD provisioner pod. CSI_ENABLE_RBD_SNAPSHOTTER: "true" diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index c9450aab2..43c0bc080 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -71,6 +71,9 @@ data: # set to false to disable deployment of snapshotter container in CephFS provisioner pod. CSI_ENABLE_CEPHFS_SNAPSHOTTER: "true" + # set to false to disable deployment of snapshotter container in NFS provisioner pod. + CSI_ENABLE_NFS_SNAPSHOTTER: "true" + # set to false to disable deployment of snapshotter container in RBD provisioner pod. CSI_ENABLE_RBD_SNAPSHOTTER: "true" diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 5c5c9750d..5f2c7ee5c 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -152,9 +152,10 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { CSIParam.EnableOIDCTokenProjection = true } - // enable RBD and CephFS snapshotter by default + // enable RBD, CephFS and NFS snapshotter by default CSIParam.EnableRBDSnapshotter = true CSIParam.EnableCephFSSnapshotter = true + CSIParam.EnableNFSSnapshotter = true if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_RBD_SNAPSHOTTER", "true"), "false") { CSIParam.EnableRBDSnapshotter = false @@ -164,6 +165,10 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { CSIParam.EnableCephFSSnapshotter = false } + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_NFS_SNAPSHOTTER", "true"), "false") { + CSIParam.EnableNFSSnapshotter = false + } + CSIParam.EnableCSIAddonsSideCar = false if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CSIADDONS", "false"), "true") { CSIParam.EnableCSIAddonsSideCar = true diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index fb99b0964..128d25c5d 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -64,6 +64,7 @@ type Param struct { EnableOMAPGenerator bool EnableRBDSnapshotter bool EnableCephFSSnapshotter bool + EnableNFSSnapshotter bool EnableCSIAddonsSideCar bool MountCustomCephConf bool EnableOIDCTokenProjection bool diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml index ad106b38b..7f4ac6d2d 100644 --- a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml @@ -21,6 +21,46 @@ spec: priorityClassName: {{ .ProvisionerPriorityClassName }} {{ end }} containers: + {{ if .EnableNFSSnapshotter }} + - name: csi-snapshotter + image: {{ .SnapshotterImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v={{ .SidecarLogLevel }}" + - "--timeout={{ .GRPCTimeout }}" + - "--leader-election=true" + - "--leader-election-namespace={{ .Namespace }}" + - "--leader-election-lease-duration=137s" + - "--leader-election-renew-deadline=107s" + - "--leader-election-retry-period=26s" + - "--extra-create-metadata=true" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + {{ end }} + - name: csi-resizer + image: {{ .ResizerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v={{ .SidecarLogLevel }}" + - "--timeout={{ .GRPCTimeout }}" + - "--leader-election=true" + - "--leader-election-namespace={{ .Namespace }}" + - "--leader-election-lease-duration=137s" + - "--leader-election-renew-deadline=107s" + - "--leader-election-retry-period=26s" + - "--handle-volume-inuse-error=false" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: csi-provisioner image: {{ .ProvisionerImage }} args: