diff --git a/Documentation/ceph-csi-snapshot.md b/Documentation/ceph-csi-snapshot.md index 07c02f216..5ed0560ce 100644 --- a/Documentation/ceph-csi-snapshot.md +++ b/Documentation/ceph-csi-snapshot.md @@ -6,15 +6,11 @@ indent: true ## Prerequisites -* Requires Kubernetes v1.17+ which supports snapshot beta. +- Rook officially supports v1/v1beta1 snapshots for kubernetes v1.17+. -* Install the new snapshot controller and snapshot beta CRD. More info can be found -[here](https://github.com/kubernetes-csi/external-snapshotter/tree/master#usage) +- Install the snapshot controller and snapshot v1/v1beta1 CRD as required. More info can be found [here](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0#usage). -Note: If the Kubernetes distributor you are using does not supports the snapshot beta, -still you can use the Alpha snapshots. refer to -[snapshot](https://github.com/rook/rook/blob/release-1.3/Documentation/ceph-csi-drivers.md#rbd-snapshots) -on how to use snapshots. +Note: If only Alpha snapshots are available, enable snapshotter in `rook-ceph-operator-config` or helm chart `values.yaml`, change the external-snapshotter image to `k8s.gcr.io/sig-storage/csi-snapshotter:v1.2.2` and refer to the [alpha snapshots documentation](https://github.com/rook/rook/blob/release-1.3/Documentation/ceph-csi-drivers.md#rbd-snapshots) * We also need a `VolumeSnapshotClass` for volume snapshot to work. The purpose of a `VolumeSnapshotClass` is defined in [the kubernetes @@ -26,6 +22,10 @@ In short, as the documentation describes it: > VolumeSnapshotClass provides a way to describe the “classes” of storage when > provisioning a volume snapshot. +## Upgrade Snapshot API + +If your Kubernetes version is updated to a newer version of the snapshot API, follow the upgrade guide [here](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0#upgrade) to upgrade from v1alpha1 to v1beta1, or v1beta1 to v1. + ## RBD Snapshots diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md index 8073d6ecb..af1b5eb69 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/ceph-upgrade.md @@ -303,7 +303,7 @@ kubectl apply -f common.yaml -f crds.yaml > The error will contain text similar to `... spec.preserveUnknownFields: Invalid value...`. If you experience this error applying the latest changes to CRDs, use `kubectl`'s `replace` command -to replace the resources followed by `apply` to verify that the resources are updated without other +to replace the resources followed by `apply` to verify that the resources are updated without other errors. ```sh kubectl replace -f crds.yaml @@ -318,6 +318,8 @@ If you have specified custom CSI images in the Rook-Ceph Operator deployment, we update to use the latest Ceph-CSI drivers. See the [CSI Version](#csi-version) section for more details. +> Note: If using snapshots, refer to the [Upgrade Snapshot API guide](ceph-csi-snapshot.md#upgrade-snapshot-api). + ## 3. Update the Rook Operator > Automatically updated if you are upgrading via the helm chart @@ -463,9 +465,9 @@ The default upstream images are included below, which you can change to your des ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.3.1" ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1" ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4" -ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2" ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2" ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1" +ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0" ``` ### Use default images @@ -487,7 +489,7 @@ k8s.gcr.io/sig-storage/csi-attacher:v3.0.2 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4 k8s.gcr.io/sig-storage/csi-resizer:v1.0.1 -k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2 +k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0 ``` ## Replace lvm mode OSDs with raw mode (if you use LV-backed PVC) diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index 7416c866a..9d0e230d8 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -131,7 +131,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.registrar.image` | Kubernetes CSI registrar image. | `k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1` | | `csi.resizer.image` | Kubernetes CSI resizer image. | `k8s.gcr.io/sig-storage/csi-resizer:v1.0.0` | | `csi.provisioner.image` | Kubernetes CSI provisioner image. | `k8s.gcr.io/sig-storage/csi-provisioner:v2.0.0` | -| `csi.snapshotter.image` | Kubernetes CSI snapshotter image. | `k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.0` | +| `csi.snapshotter.image` | Kubernetes CSI snapshotter image. | `k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0` | | `csi.attacher.image` | Kubernetes CSI Attacher image. | `k8s.gcr.io/sig-storage/csi-attacher:v3.0.0` | | `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Pods. | | | `csi.rbdPodLabels` | Labels to add to the CSI RBD Pods. | | diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 2a658ff93..a9f79d524 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -274,7 +274,7 @@ csi: #provisioner: #image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4 #snapshotter: - #image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2 + #image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0 #attacher: #image: k8s.gcr.io/sig-storage/csi-attacher:v3.0.2 #resizer: diff --git a/cluster/examples/kubernetes/ceph/csi/cephfs/snapshot.yaml b/cluster/examples/kubernetes/ceph/csi/cephfs/snapshot.yaml index 84907b975..2c734c511 100644 --- a/cluster/examples/kubernetes/ceph/csi/cephfs/snapshot.yaml +++ b/cluster/examples/kubernetes/ceph/csi/cephfs/snapshot.yaml @@ -1,5 +1,8 @@ --- -apiVersion: snapshot.storage.k8s.io/v1beta1 +# 1.17 <= K8s <= v1.19 +# apiVersion: snapshot.storage.k8s.io/v1beta1 +# K8s >= v1.20 +apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshot metadata: name: cephfs-pvc-snapshot diff --git a/cluster/examples/kubernetes/ceph/csi/cephfs/snapshotclass.yaml b/cluster/examples/kubernetes/ceph/csi/cephfs/snapshotclass.yaml index e2aaac009..eabdb2920 100644 --- a/cluster/examples/kubernetes/ceph/csi/cephfs/snapshotclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/cephfs/snapshotclass.yaml @@ -1,5 +1,9 @@ --- -apiVersion: snapshot.storage.k8s.io/v1beta1 +# apiVersion: snapshot.storage.k8s.io/v1alpha1 +# 1.17 <= K8s <= v1.19 +# apiVersion: snapshot.storage.k8s.io/v1beta1 +# K8s >= v1.20 +apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotClass metadata: name: csi-cephfsplugin-snapclass diff --git a/cluster/examples/kubernetes/ceph/csi/rbd/snapshot.yaml b/cluster/examples/kubernetes/ceph/csi/rbd/snapshot.yaml index a7127ab57..9c27b410d 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbd/snapshot.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbd/snapshot.yaml @@ -1,5 +1,8 @@ --- -apiVersion: snapshot.storage.k8s.io/v1beta1 +# 1.17 <= K8s <= v1.19 +# apiVersion: snapshot.storage.k8s.io/v1beta1 +# K8s >= v1.20 +apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshot metadata: name: rbd-pvc-snapshot diff --git a/cluster/examples/kubernetes/ceph/csi/rbd/snapshotclass.yaml b/cluster/examples/kubernetes/ceph/csi/rbd/snapshotclass.yaml index df7896822..2ed388456 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbd/snapshotclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbd/snapshotclass.yaml @@ -1,5 +1,8 @@ --- -apiVersion: snapshot.storage.k8s.io/v1beta1 +# 1.17 <= K8s <= v1.19 +# apiVersion: snapshot.storage.k8s.io/v1beta1 +# K8s >= v1.20 +apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotClass metadata: name: csi-rbdplugin-snapclass diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 21112cae1..b74d20b2c 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -149,7 +149,7 @@ data: # ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1" # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1" # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4" - # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2" + # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0" # ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2" # (Optional) set user created priorityclassName for csi plugin pods. diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 1e0212e79..443862e69 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -73,7 +73,7 @@ data: # ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1" # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1" # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4" - # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2" + # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0" # ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2" # (Optional) set user created priorityclassName for csi plugin pods. diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 4afa436ff..ab90597dd 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -112,7 +112,7 @@ var ( DefaultRegistrarImage = "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1" DefaultProvisionerImage = "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4" DefaultAttacherImage = "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2" - DefaultSnapshotterImage = "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2" + DefaultSnapshotterImage = "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0" DefaultResizerImage = "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1" DefaultVolumeReplicationImage = "quay.io/csiaddons/volumereplication-operator:v0.1.0" ) @@ -122,6 +122,7 @@ const ( ProvDeploymentSuppVersion = "14" kubeMinVerForFilesystemRestore = "15" kubeMinVerForBlockRestore = "16" + kubeMinVerForSnapshot = "17" // toleration and node affinity provisionerTolerationsEnv = "CSI_PROVISIONER_TOLERATIONS" @@ -290,8 +291,12 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter if strings.EqualFold(enableOMAPGenerator, "true") { tp.EnableOMAPGenerator = true } - // enable RBD snapshotter by default - tp.EnableRBDSnapshotter = true + + // if k8s >= v1.17 enable RBD and CephFS snapshotter by default + if ver.Major == KubeMinMajor && ver.Minor >= kubeMinVerForSnapshot { + tp.EnableRBDSnapshotter = true + tp.EnableCephFSSnapshotter = true + } enableRBDSnapshotter, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_RBD_SNAPSHOTTER", "true") if err != nil { return errors.Wrap(err, "failed to load CSI_ENABLE_RBD_SNAPSHOTTER setting") @@ -299,8 +304,6 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter if strings.EqualFold(enableRBDSnapshotter, "false") { tp.EnableRBDSnapshotter = false } - // enable CephFS snapshotter by default - tp.EnableCephFSSnapshotter = true enableCephFSSnapshotter, err := k8sutil.GetOperatorSetting(clientset, controllerutil.OperatorSettingConfigMapName, "CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true") if err != nil { return errors.Wrap(err, "failed to load CSI_ENABLE_CEPHFS_SNAPSHOTTER setting") diff --git a/tests/framework/utils/snapshot.go b/tests/framework/utils/snapshot.go index fd278eb40..b333ec2f6 100644 --- a/tests/framework/utils/snapshot.go +++ b/tests/framework/utils/snapshot.go @@ -27,7 +27,7 @@ import ( const ( // snapshotterVersion from which the snapshotcontroller and CRD will be // installed - snapshotterVersion = "v3.0.0" + snapshotterVersion = "v4.0.0" repoURL = "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter" rbacPath = "deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" controllerPath = "deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"