ceph: update snapshot APIs from v1beta1 to v1

This commit updates external-snapshotter version to
v4.0.0 which supports snapshots v1.
Rook now defaults to enabling RBD and CephFS snapshotter
for K8s >= v1.17 and disabling it for K8s <= v1.16.
Supporting changes in documents and examples yaml files
are made.

Signed-off-by: Rakshith R <rar@redhat.com>
This commit is contained in:
Rakshith R committed 2021-05-05 21:04:10 +05:30
1 parent 40e6633280
commit 7987e1b8a2
12 files changed
+42 -24

No files matched your search

+7 -7
View File
@@ -6,15 +6,11 @@ indent: true
## Prerequisites
* Requires Kubernetes v1.17+ which supports snapshot beta.
- Rook officially supports v1/v1beta1 snapshots for kubernetes v1.17+.
* Install the new snapshot controller and snapshot beta CRD. More info can be found
[here](https://github.com/kubernetes-csi/external-snapshotter/tree/master#usage)
- Install the snapshot controller and snapshot v1/v1beta1 CRD as required. More info can be found [here](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0#usage).
Note: If the Kubernetes distributor you are using does not supports the snapshot beta,
still you can use the Alpha snapshots. refer to
[snapshot](https://github.com/rook/rook/blob/release-1.3/Documentation/ceph-csi-drivers.md#rbd-snapshots)
on how to use snapshots.
Note: If only Alpha snapshots are available, enable snapshotter in `rook-ceph-operator-config` or helm chart `values.yaml`, change the external-snapshotter image to `k8s.gcr.io/sig-storage/csi-snapshotter:v1.2.2` and refer to the [alpha snapshots documentation](https://github.com/rook/rook/blob/release-1.3/Documentation/ceph-csi-drivers.md#rbd-snapshots)
* We also need a `VolumeSnapshotClass` for volume snapshot to work. The purpose of a `VolumeSnapshotClass` is
defined in [the kubernetes
@@ -26,6 +22,10 @@ In short, as the documentation describes it:
> VolumeSnapshotClass provides a way to describe the “classes” of storage when
> provisioning a volume snapshot.
## Upgrade Snapshot API
If your Kubernetes version is updated to a newer version of the snapshot API, follow the upgrade guide [here](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0#upgrade) to upgrade from v1alpha1 to v1beta1, or v1beta1 to v1.
## RBD Snapshots
+5 -3
View File
@@ -303,7 +303,7 @@ kubectl apply -f common.yaml -f crds.yaml
> The error will contain text similar to `... spec.preserveUnknownFields: Invalid value...`.
If you experience this error applying the latest changes to CRDs, use `kubectl`'s `replace` command
to replace the resources followed by `apply` to verify that the resources are updated without other
to replace the resources followed by `apply` to verify that the resources are updated without other
errors.
```sh
kubectl replace -f crds.yaml
@@ -318,6 +318,8 @@ If you have specified custom CSI images in the Rook-Ceph Operator deployment, we
update to use the latest Ceph-CSI drivers. See the [CSI Version](#csi-version) section for more
details.
> Note: If using snapshots, refer to the [Upgrade Snapshot API guide](ceph-csi-snapshot.md#upgrade-snapshot-api).
## 3. Update the Rook Operator
> Automatically updated if you are upgrading via the helm chart
@@ -463,9 +465,9 @@ The default upstream images are included below, which you can change to your des
ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.3.1"
ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4"
ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2"
ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2"
ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1"
ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0"
```
### Use default images
@@ -487,7 +489,7 @@ k8s.gcr.io/sig-storage/csi-attacher:v3.0.2
k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4
k8s.gcr.io/sig-storage/csi-resizer:v1.0.1
k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2
k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
```
## Replace lvm mode OSDs with raw mode (if you use LV-backed PVC)
+1 -1
View File
@@ -131,7 +131,7 @@ The following tables lists the configurable parameters of the rook-operator char
| `csi.registrar.image` | Kubernetes CSI registrar image. | `k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1` |
| `csi.resizer.image` | Kubernetes CSI resizer image. | `k8s.gcr.io/sig-storage/csi-resizer:v1.0.0` |
| `csi.provisioner.image` | Kubernetes CSI provisioner image. | `k8s.gcr.io/sig-storage/csi-provisioner:v2.0.0` |
| `csi.snapshotter.image` | Kubernetes CSI snapshotter image. | `k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.0` |
| `csi.snapshotter.image` | Kubernetes CSI snapshotter image. | `k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0` |
| `csi.attacher.image` | Kubernetes CSI Attacher image. | `k8s.gcr.io/sig-storage/csi-attacher:v3.0.0` |
| `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Pods. | <none> |
| `csi.rbdPodLabels` | Labels to add to the CSI RBD Pods. | <none> |
+1 -1
View File
@@ -274,7 +274,7 @@ csi:
#provisioner:
#image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4
#snapshotter:
#image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2
#image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
#attacher:
#image: k8s.gcr.io/sig-storage/csi-attacher:v3.0.2
#resizer:
@@ -1,5 +1,8 @@
---
apiVersion: snapshot.storage.k8s.io/v1beta1
# 1.17 <= K8s <= v1.19
# apiVersion: snapshot.storage.k8s.io/v1beta1
# K8s >= v1.20
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshot
metadata:
name: cephfs-pvc-snapshot
@@ -1,5 +1,9 @@
---
apiVersion: snapshot.storage.k8s.io/v1beta1
# apiVersion: snapshot.storage.k8s.io/v1alpha1
# 1.17 <= K8s <= v1.19
# apiVersion: snapshot.storage.k8s.io/v1beta1
# K8s >= v1.20
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
name: csi-cephfsplugin-snapclass
@@ -1,5 +1,8 @@
---
apiVersion: snapshot.storage.k8s.io/v1beta1
# 1.17 <= K8s <= v1.19
# apiVersion: snapshot.storage.k8s.io/v1beta1
# K8s >= v1.20
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshot
metadata:
name: rbd-pvc-snapshot
@@ -1,5 +1,8 @@
---
apiVersion: snapshot.storage.k8s.io/v1beta1
# 1.17 <= K8s <= v1.19
# apiVersion: snapshot.storage.k8s.io/v1beta1
# K8s >= v1.20
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
name: csi-rbdplugin-snapclass
@@ -149,7 +149,7 @@ data:
# ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
# ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1"
# ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4"
# ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2"
# ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0"
# ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2"
# (Optional) set user created priorityclassName for csi plugin pods.
@@ -73,7 +73,7 @@ data:
# ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
# ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1"
# ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4"
# ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2"
# ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0"
# ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2"
# (Optional) set user created priorityclassName for csi plugin pods.
+8 -5
View File
@@ -112,7 +112,7 @@ var (
DefaultRegistrarImage = "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
DefaultProvisionerImage = "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4"
DefaultAttacherImage = "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2"
DefaultSnapshotterImage = "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2"
DefaultSnapshotterImage = "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0"
DefaultResizerImage = "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1"
DefaultVolumeReplicationImage = "quay.io/csiaddons/volumereplication-operator:v0.1.0"
)
@@ -122,6 +122,7 @@ const (
ProvDeploymentSuppVersion = "14"
kubeMinVerForFilesystemRestore = "15"
kubeMinVerForBlockRestore = "16"
kubeMinVerForSnapshot = "17"
// toleration and node affinity
provisionerTolerationsEnv = "CSI_PROVISIONER_TOLERATIONS"
@@ -290,8 +291,12 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
if strings.EqualFold(enableOMAPGenerator, "true") {
tp.EnableOMAPGenerator = true
}
// enable RBD snapshotter by default
tp.EnableRBDSnapshotter = true
// if k8s >= v1.17 enable RBD and CephFS snapshotter by default
if ver.Major == KubeMinMajor && ver.Minor >= kubeMinVerForSnapshot {
tp.EnableRBDSnapshotter = true
tp.EnableCephFSSnapshotter = true
}
enableRBDSnapshotter, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_RBD_SNAPSHOTTER", "true")
if err != nil {
return errors.Wrap(err, "failed to load CSI_ENABLE_RBD_SNAPSHOTTER setting")
@@ -299,8 +304,6 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
if strings.EqualFold(enableRBDSnapshotter, "false") {
tp.EnableRBDSnapshotter = false
}
// enable CephFS snapshotter by default
tp.EnableCephFSSnapshotter = true
enableCephFSSnapshotter, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true")
if err != nil {
return errors.Wrap(err, "failed to load CSI_ENABLE_CEPHFS_SNAPSHOTTER setting")
+1 -1
View File
@@ -27,7 +27,7 @@ import (
const (
// snapshotterVersion from which the snapshotcontroller and CRD will be
// installed
snapshotterVersion = "v3.0.0"
snapshotterVersion = "v4.0.0"
repoURL = "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter"
rbacPath = "deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml"
controllerPath = "deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"