From e622c52f3fd3583068e8a4cebab5b1549e68ca11 Mon Sep 17 00:00:00 2001 From: Niraj Yadav Date: Thu, 12 Sep 2024 18:25:09 +0530 Subject: [PATCH] docs: add documentation for volumegroupsnapshot This PR adds the sample YAMLs and documentation for the VolumeGroupSnapshot feature. Signed-off-by: Niraj Yadav (cherry picked from commit 1e3d08f11409dcf3e1f0119fe9388ba840bb98d6) --- .../Storage-Configuration/Ceph-CSI/.pages | 1 + .../Ceph-CSI/ceph-csi-snapshot.md | 5 - .../ceph-csi-volume-group-snapshot.md | 109 ++++++++++++++++++ deploy/examples/csi/cephfs/groupsnapshot.yaml | 13 +++ .../csi/cephfs/groupsnapshotclass.yaml | 15 +++ deploy/examples/csi/cephfs/pvc.yaml | 2 + 6 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-group-snapshot.md create mode 100644 deploy/examples/csi/cephfs/groupsnapshot.yaml create mode 100644 deploy/examples/csi/cephfs/groupsnapshotclass.yaml diff --git a/Documentation/Storage-Configuration/Ceph-CSI/.pages b/Documentation/Storage-Configuration/Ceph-CSI/.pages index a117ff7dd..47db28e1c 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/.pages +++ b/Documentation/Storage-Configuration/Ceph-CSI/.pages @@ -1,6 +1,7 @@ nav: - ceph-csi-drivers.md - ceph-csi-snapshot.md + - ceph-csi-volume-group-snapshot.md - ceph-csi-volume-clone.md - custom-images.md - ... diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md index 1a8d888d1..c436b7198 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md @@ -178,8 +178,3 @@ kubectl delete -f deploy/examples/csi/cephfs/pvc-restore.yaml kubectl delete -f deploy/examples/csi/cephfs/snapshot.yaml kubectl delete -f deploy/examples/csi/cephfs/snapshotclass.yaml ``` - -## Limitations - -- There is a limit of 400 snapshots per cephFS filesystem. -- The PVC cannot be deleted if it has snapshots. make sure all the snapshots on the PVC are deleted before you delete the PVC. diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-group-snapshot.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-group-snapshot.md new file mode 100644 index 000000000..33d58b807 --- /dev/null +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-group-snapshot.md @@ -0,0 +1,109 @@ +--- +title: Volume Group Snapshots +--- + +Ceph provides the ability to create crash-consistent snapshots of multiple volumes. +A group snapshot represents “copies” from multiple volumes that are taken at the same point in time. +A group snapshot can be used either to rehydrate new volumes (pre-populated with the snapshot data) +or to restore existing volumes to a previous state (represented by the snapshots) + + +## Prerequisites + +- Install the [snapshot controller, volume group snapshot and snapshot CRDs](https://github.com/kubernetes-csi/external-snapshotter/tree/master#usage), +refer to VolumeGroupSnapshot documentation +[here](https://github.com/kubernetes-csi/external-snapshotter/tree/master#volume-group-snapshot-support) for more details. + +- A `VolumeGroupSnapshotClass` is needed for the volume group snapshot to work. The purpose of a `VolumeGroupSnapshotClass` is +defined in [the kubernetes +documentation](https://kubernetes.io/blog/2023/05/08/kubernetes-1-27-volume-group-snapshot-alpha/). +In short, as the documentation describes it: + +!!! info + Created by cluster administrators to describe how volume group snapshots + should be created. including the driver information, the deletion policy, etc. + +## Volume Group Snapshots + +### CephFS VolumeGroupSnapshotClass + +In [VolumeGroupSnapshotClass](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/groupsnapshotclass.yaml), +the `csi.storage.k8s.io/group-snapshotter-secret-name` parameter should reference the +name of the secret created for the cephfs-plugin. + +In the `VolumeGroupSnapshotClass`, update the value of the `clusterID` field to match the namespace +that Rook is running in. When Ceph CSI is deployed by Rook, the operator will automatically +maintain a configmap whose contents will match this key. By default this is +"rook-ceph". + +```console +kubectl create -f deploy/examples/csi/cephfs/groupsnapshotclass.yaml +``` + +### CephFS VolumeGroupSnapshot + +In [VolumeGroupSnapshot](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/groupsnapshot.yaml), +`volumeGroupSnapshotClassName` should be the name of the `VolumeGroupSnapshotClass` +previously created. The labels inside `matchLabels` should be present on the +PVCs that are already created by the CephFS CSI driver. + +```console +kubectl create -f deploy/examples/csi/cephfs/groupsnapshot.yaml +``` + +### Verify CephFS GroupSnapshot Creation + +```console +$ kubectl get volumegroupsnapshotclass +NAME DRIVER DELETIONPOLICY AGE +csi-cephfsplugin-groupsnapclass rook-ceph.cephfs.csi.ceph.com Delete 21m +``` + +```console +$ kubectl get volumegroupsnapshot +NAME READYTOUSE VOLUMEGROUPSNAPSHOTCLASS VOLUMEGROUPSNAPSHOTCONTENT CREATIONTIME AGE +cephfs-groupsnapshot true csi-cephfsplugin-groupsnapclass groupsnapcontent-d13f4d95-8822-4729-9586-4f222a3f788e 5m37s 5m39s +``` + +The snapshot will be ready to restore to a new PVC when `READYTOUSE` field of the +`volumegroupsnapshot` is set to true. + +### Restore the CephFS volume group snapshot to a new PVC + +Find the name of the snapshots created by the `VolumeGroupSnapshot` first by running: + +```console +$ kubectl get volumegroupsnapshot/cephfs-groupsnapshot -o=jsonpath='{range .status.pvcVolumeSnapshotRefList[*]}PVC: {.persistentVolumeClaimRef.name}, Snapshot: {.volumeSnapshotRef.name}{"\n"}{end}' +PVC: cephfs-pvc, Snapshot: snapshot-9d21b143904c10f49ddc92664a7e8fe93c23387d0a88549c14337484ebaf1011-2024-09-12-3.49.13 +``` + +It will list the PVC's name followed by its snapshot name. + +In +[pvc-restore](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/pvc-restore.yaml), +`dataSource` should be one of the `Snapshot` that we just +found. The `dataSource` kind should be the `VolumeSnapshot`. + +Create a new PVC from the snapshot + +```console +kubectl create -f deploy/examples/csi/cephfs/pvc-restore.yaml +``` + +### Verify CephFS Restore PVC Creation + +```console +$ kubectl get pvc +cephfs-pvc Bound pvc-9ae60bf9-4931-4f9a-9de1-7f45f31fe4da 1Gi RWO rook-cephfs 171m +cephfs-pvc-restore Bound pvc-b4b73cbb-5061-48c7-9ac8-e1202508cf97 1Gi RWO rook-cephfs 46s +``` + +## CephFS volume group snapshot resource Cleanup + +To clean the resources created by this example, run the following: + +```console +kubectl delete -f deploy/examples/csi/cephfs/pvc-restore.yaml +kubectl delete -f deploy/examples/csi/cephfs/groupsnapshot.yaml +kubectl delete -f deploy/examples/csi/cephfs/groupsnapshotclass.yaml +``` diff --git a/deploy/examples/csi/cephfs/groupsnapshot.yaml b/deploy/examples/csi/cephfs/groupsnapshot.yaml new file mode 100644 index 000000000..98a8a6d46 --- /dev/null +++ b/deploy/examples/csi/cephfs/groupsnapshot.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: groupsnapshot.storage.k8s.io/v1alpha1 +kind: VolumeGroupSnapshot +metadata: + name: cephfs-groupsnapshot +spec: + source: + selector: + matchLabels: + # The PVCs require this label for them to be + # included in the VolumeGroupSnapshot + group: snapshot-test + volumeGroupSnapshotClassName: csi-cephfsplugin-groupsnapclass diff --git a/deploy/examples/csi/cephfs/groupsnapshotclass.yaml b/deploy/examples/csi/cephfs/groupsnapshotclass.yaml new file mode 100644 index 000000000..fe8d2e775 --- /dev/null +++ b/deploy/examples/csi/cephfs/groupsnapshotclass.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: groupsnapshot.storage.k8s.io/v1alpha1 +kind: VolumeGroupSnapshotClass +metadata: + name: csi-cephfsplugin-groupsnapclass +driver: rook-ceph.cephfs.csi.ceph.com # csi-provisioner-name +parameters: + # Specify a string that identifies your cluster. Ceph CSI supports any + # unique string. When Ceph CSI is deployed by Rook use the Rook namespace, + # for example "rook-ceph". + clusterID: rook-ceph # namespace: cluster + fsName: myfs + csi.storage.k8s.io/group-snapshotter-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/group-snapshotter-secret-namespace: rook-ceph +deletionPolicy: Delete diff --git a/deploy/examples/csi/cephfs/pvc.yaml b/deploy/examples/csi/cephfs/pvc.yaml index 0f6addb69..cf3057de3 100644 --- a/deploy/examples/csi/cephfs/pvc.yaml +++ b/deploy/examples/csi/cephfs/pvc.yaml @@ -3,6 +3,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: cephfs-pvc + labels: + group: snapshot-test spec: accessModes: - ReadWriteOnce