From cf161848997a897dad8e6ea93fcb464bccba67d8 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 21 Jan 2020 13:34:23 +0530 Subject: [PATCH] CSI: Add support for CSI volume expansion This PR makes the necessary changes required for volume expansion support for both cephfs and rbd. Signed-off-by: Madhu Rajanna --- Documentation/ceph-csi-drivers.md | 21 +++++++++++++++++-- Documentation/helm-operator.md | 1 + .../rook-ceph/templates/deployment.yaml | 6 ++++++ cluster/charts/rook-ceph/values.yaml | 2 ++ cluster/examples/kubernetes/ceph/common.yaml | 10 +++++++-- .../ceph/csi/cephfs/storageclass.yaml | 3 +++ .../ceph/csi/rbd/storageclass-test.yaml | 3 +++ .../kubernetes/ceph/csi/rbd/storageclass.yaml | 3 +++ .../csi-cephfsplugin-provisioner-dep.yaml | 15 +++++++++++++ .../rbd/csi-rbdplugin-provisioner-dep.yaml | 15 +++++++++++++ .../kubernetes/ceph/operator-openshift.yaml | 2 ++ .../examples/kubernetes/ceph/operator.yaml | 2 ++ .../ceph/upgrade-from-v1.2-apply.yaml | 11 +++++++--- pkg/operator/ceph/csi/spec.go | 7 +++++++ 14 files changed, 94 insertions(+), 7 deletions(-) diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/ceph-csi-drivers.md index ea4e8c873..09f3f617d 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/ceph-csi-drivers.md @@ -19,8 +19,6 @@ For documentation on consuming the storage: * RBD: See the [Block Storage](ceph-block.md) topic * CephFS: See the [Shared Filesystem](ceph-filesystem.md) topic -The remainder of this topic is on the snapshotting feature of the RBD driver. - ## RBD Snapshots Since this feature is still in [alpha @@ -138,3 +136,22 @@ csi_liveness 1 Check the [monitoring doc](ceph-monitoring.md) to see how to integrate CSI liveness and grpc metrics into ceph monitoring. + +## Dynamically Expand Volume + +### Prerequisite + +* For filesystem resize to be supported for your Kubernetes cluster, the + kubernetes version running in your cluster should be >= v1.15 and for block + volume resize support the Kubernetes version should be >= v1.16. Also, + `ExpandCSIVolumes` feature gate has to be enabled for the volume resize + functionality to work. + +To expand the PVC the controlling StorageClass must have `allowVolumeExpansion` +set to `true`. `csi.storage.k8s.io/controller-expand-secret-name` and +`csi.storage.k8s.io/controller-expand-secret-namespace` values set in +storageclass. Now expand the PVC by editing the PVC +`pvc.spec.resource.requests.storage` to a higher values than the current size. +Once PVC is expanded on backend and same is reflected size is reflected on +application mountpoint, the status capacity `pvc.status.capacity.storage` of +PVC will be updated to new size. diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index 7e737943b..1c3fbc06f 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -134,6 +134,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.rbdPluginUpdateStrategy` | CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `OnDelete` | | `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `OnDelete` | | `csi.registrar.image` | Kubernetes CSI registrar image. | `quay.io/k8scsi/csi-node-driver-registrar:v1.2.0` | +| `csi.resizer.image` | Kubernetes CSI resizer image. | `quay.io/k8scsi/csi-resizer:v0.4.0` | | `csi.provisioner.image` | Kubernetes CSI provisioner image. | `quay.io/k8scsi/csi-provisioner:v1.4.0` | | `csi.snapshotter.image` | Kubernetes CSI snapshotter image. | `quay.io/k8scsi/csi-snapshotter:v1.2.2` | | `csi.attacher.image` | Kubernetes CSI Attacher image. | `quay.io/k8scsi/csi-attacher:v2.1.0` | diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 6c62fc7cf..010e177aa 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -152,6 +152,12 @@ spec: value: {{ .Values.csi.attacher.image | quote }} {{- end }} {{- end }} +{{- if .Values.csi.resizer }} +{{- if .Values.csi.resizer.image }} + - name: ROOK_CSI_RESIZER_IMAGE + value: {{ .Values.csi.resizer.image | quote }} +{{- end }} +{{- end }} {{- if .Values.csi.provisionerTolerations }} - name: CSI_PROVISIONER_TOLERATIONS value: {{ toYaml .Values.csi.provisionerTolerations | quote }} diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index daca0ff7d..7ee24c0e4 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -98,6 +98,8 @@ csi: #image: quay.io/k8scsi/csi-snapshotter:v1.2.2 #attacher: #image: quay.io/k8scsi/csi-attacher:v2.1.0 + #resizer: + #image: quay.io/k8scsi/csi-resizer:v0.4.0 enableFlexDriver: false enableDiscoveryDaemon: true diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 67f38cc88..985268f1b 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -1377,7 +1377,7 @@ rules: verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] @@ -1393,6 +1393,9 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] # OLM: END CSI CEPHFS CLUSTER ROLE # OLM: BEGIN CSI CEPHFS CLUSTER ROLEBINDING --- @@ -1557,7 +1560,7 @@ rules: verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] @@ -1588,6 +1591,9 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots/status"] verbs: ["update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] # OLM: END CSI RBD CLUSTER ROLE # OLM: BEGIN CSI RBD CLUSTER ROLEBINDING --- diff --git a/cluster/examples/kubernetes/ceph/csi/cephfs/storageclass.yaml b/cluster/examples/kubernetes/ceph/csi/cephfs/storageclass.yaml index 7b6de0397..5299dfedd 100644 --- a/cluster/examples/kubernetes/ceph/csi/cephfs/storageclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/cephfs/storageclass.yaml @@ -22,6 +22,8 @@ parameters: # in the same namespace as the cluster. csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph @@ -30,6 +32,7 @@ parameters: # or by setting the default mounter explicitly via --volumemounter command-line argument. # mounter: kernel reclaimPolicy: Delete +allowVolumeExpansion: true mountOptions: # uncomment the following line for debugging #- debug diff --git a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-test.yaml b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-test.yaml index 57903e025..0acbd0b50 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-test.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-test.yaml @@ -31,6 +31,8 @@ parameters: # in the same namespace as the cluster. csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # Specify the filesystem type of the volume. If not specified, csi-provisioner @@ -38,4 +40,5 @@ parameters: csi.storage.k8s.io/fstype: ext4 # uncomment the following to use rbd-nbd as mounter on supported nodes #mounter: rbd-nbd +allowVolumeExpansion: true reclaimPolicy: Delete diff --git a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml index 7fdab750c..f9e2d6c00 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml @@ -31,6 +31,8 @@ parameters: # in the same namespace as the cluster. csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # Specify the filesystem type of the volume. If not specified, csi-provisioner @@ -41,4 +43,5 @@ parameters: # issue that causes the mount to be disconnected. You will need to follow special upgrade steps # to restart your application pods. Therefore, this option is not recommended. #mounter: rbd-nbd +allowVolumeExpansion: true reclaimPolicy: Delete diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index 80a2a2c8e..fdd06f4fe 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -31,6 +31,21 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + - name: csi-resizer + image: {{ .ResizerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--csiTimeout=150s" + - "--leader-election" + - "--leader-election-namespace={{ .Namespace }}" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: csi-provisioner image: {{ .ProvisionerImage }} args: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 2c8e4b887..42e4958a6 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -33,6 +33,21 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + - name: csi-resizer + image: {{ .ResizerImage }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--csiTimeout=150s" + - "--leader-election" + - "--leader-election-namespace={{ .Namespace }}" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: csi-rbdplugin-attacher image: {{ .AttacherImage }} args: diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 527605f90..ec767de99 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -247,6 +247,8 @@ spec: # value: "quay.io/cephcsi/cephcsi:v2.0.0" #- name: ROOK_CSI_REGISTRAR_IMAGE # value: "quay.io/k8scsi/csi-node-driver-registrar:v1.2.0" + #- name: ROOK_CSI_RESIZER_IMAGE + # value: "quay.io/k8scsi/csi-resizer:v0.4.0" #- name: ROOK_CSI_PROVISIONER_IMAGE # value: "quay.io/k8scsi/csi-provisioner:v1.4.0" #- name: ROOK_CSI_SNAPSHOTTER_IMAGE diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index e79377e0a..67f26a3ac 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -205,6 +205,8 @@ spec: # value: "quay.io/cephcsi/cephcsi:v2.0.0" #- name: ROOK_CSI_REGISTRAR_IMAGE # value: "quay.io/k8scsi/csi-node-driver-registrar:v1.2.0" + #- name: ROOK_CSI_RESIZER_IMAGE + # value: "quay.io/k8scsi/csi-resizer:v0.4.0" #- name: ROOK_CSI_PROVISIONER_IMAGE # value: "quay.io/k8scsi/csi-provisioner:v1.4.0" #- name: ROOK_CSI_SNAPSHOTTER_IMAGE diff --git a/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml b/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml index 2dfa5a9f5..71cb35b64 100644 --- a/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml +++ b/cluster/examples/kubernetes/ceph/upgrade-from-v1.2-apply.yaml @@ -10,7 +10,7 @@ rules: verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] @@ -26,6 +26,9 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -39,7 +42,7 @@ rules: verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] @@ -70,4 +73,6 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots/status"] verbs: ["update"] - + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 5436c5518..271d8196b 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -40,6 +40,7 @@ type Param struct { ProvisionerImage string AttacherImage string SnapshotterImage string + ResizerImage string DriverNamePrefix string EnableSnapshotter string EnableCSIGRPCMetrics string @@ -95,6 +96,7 @@ var ( DefaultProvisionerImage = "quay.io/k8scsi/csi-provisioner:v1.4.0" DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v2.1.0" DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.2.2" + defaultResizerImage = "quay.io/k8scsi/csi-resizer:v0.4.0" ) const ( @@ -243,6 +245,11 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers deployProvSTS = true } + tp.ResizerImage = os.Getenv("ROOK_CSI_RESIZER_IMAGE") + if tp.ResizerImage == "" { + tp.ResizerImage = defaultResizerImage + } + if EnableRBD { rbdPlugin, err = templateToDaemonSet("rbdplugin", RBDPluginTemplatePath, tp) if err != nil {