From 345f099bbc4ffdba4bd9b25c7eec9a4e7993d3e5 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 20 Sep 2019 10:49:58 +0530 Subject: [PATCH] Make kubelet path configurable in operator currently CSI templates have hardcoded kubelet path which will be used for mounting PVC,this will not work if kubelet is configured to use different path, with this PR kubelet path can be configured for CSI. Fixes: #3921 Signed-off-by: Madhu Rajanna --- Documentation/ceph-upgrade.md | 9 +++++++++ cluster/charts/rook-ceph/templates/deployment.yaml | 6 ++++++ cluster/charts/rook-ceph/values.yaml | 1 + .../cephfs/csi-cephfsplugin-provisioner-dep.yaml | 2 +- .../cephfs/csi-cephfsplugin-provisioner-sts.yaml | 2 +- .../ceph/csi/template/cephfs/csi-cephfsplugin.yaml | 14 +++++++------- .../rbd/csi-rbdplugin-provisioner-sts.yaml | 2 +- .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 14 +++++++------- cluster/examples/kubernetes/ceph/operator.yaml | 4 +++- cmd/rook/ceph/operator.go | 2 ++ pkg/operator/ceph/csi/spec.go | 4 ++++ 11 files changed, 42 insertions(+), 18 deletions(-) diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md index 21637187f..31a874764 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/ceph-upgrade.md @@ -270,6 +270,15 @@ below, which you should change to match where your images are located. You can also remove the `ROOK_CSI_CEPHFS_IMAGE` and `ROOK_CSI_RBD_IMAGE` `env` variables that are no longer used in Rook. +If you have configured the kubelet to use other than `/var/lib/kubelet` please +add below to the operator `env` variables. + +```yaml +env: + - name: ROOK_CSI_KUBELET_DIR_PATH + value: "/kubelet/path" +``` + At the same time you edit the CSI driver settings, go ahead and update the operator deployment image: ```yaml image: rook/ceph:v1.1.0 diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 801be12c9..5a5f95b59 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -101,6 +101,12 @@ spec: - name: ROOK_CSI_ENABLE_CEPHFS value: {{ .Values.csi.enableCephfsDriver | quote }} {{- end }} +{{- if .Values.csi }} +{{- if .Values.csi.kubeletDirPath }} + - name: ROOK_CSI_KUBELET_DIR_PATH + value: {{ .Values.csi.kubeletDirPath | quote }} +{{- end }} +{{- end }} {{- if .Values.csi }} - name: ROOK_CSI_ENABLE_GRPC_METRICS value: {{ .Values.csi.enableGrpcMetrics | quote }} diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 9b12e3019..bee02e908 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -59,6 +59,7 @@ csi: enableRbdDriver: true enableCephfsDriver: true enableGrpcMetrics: true + #kubeletDirPath: /var/lib/kubelet #cephcsi: #image: quay.io/cephcsi/cephcsi:v1.2.0 #registrar: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index 74eb24125..fdba6063d 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -120,7 +120,7 @@ spec: volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com" type: DirectoryOrCreate - name: host-sys hostPath: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml index 4882a41c9..9bef3f6d2 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml @@ -115,7 +115,7 @@ spec: volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com" type: DirectoryOrCreate - name: host-sys hostPath: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 15c4d72e2..e6b2f168f 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -24,7 +24,7 @@ spec: args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/csi.sock" + - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/csi.sock" lifecycle: preStop: exec: @@ -79,10 +79,10 @@ spec: - name: plugin-dir mountPath: /csi - name: csi-plugins-dir - mountPath: /var/lib/kubelet/plugins + mountPath: "{{ .KubeletDirPath }}/plugins" mountPropagation: "Bidirectional" - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods + mountPath: "{{ .KubeletDirPath }}/pods" mountPropagation: "Bidirectional" - name: host-sys mountPath: /sys @@ -120,19 +120,19 @@ spec: volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/ + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/" type: DirectoryOrCreate - name: csi-plugins-dir hostPath: - path: /var/lib/kubelet/plugins + path: "{{ .KubeletDirPath }}/plugins" type: Directory - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: "{{ .KubeletDirPath }}/plugins_registry/" type: Directory - name: pods-mount-dir hostPath: - path: /var/lib/kubelet/pods + path: "{{ .KubeletDirPath }}/pods" type: Directory - name: host-sys hostPath: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml index be3b2c4e0..83cdb64da 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml @@ -146,7 +146,7 @@ spec: path: /lib/modules - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com" type: DirectoryOrCreate - name: ceph-csi-config configMap: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index 42f6554fb..932c2b183 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -25,7 +25,7 @@ spec: args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com/csi.sock" + - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com/csi.sock" lifecycle: preStop: exec: @@ -81,10 +81,10 @@ spec: - name: plugin-dir mountPath: /csi - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods + mountPath: "{{ .KubeletDirPath }}/pods" mountPropagation: "Bidirectional" - name: plugin-mount-dir - mountPath: /var/lib/kubelet/plugins + mountPath: "{{ .KubeletDirPath }}/plugins" mountPropagation: "Bidirectional" - mountPath: /dev name: host-dev @@ -122,19 +122,19 @@ spec: volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com" type: DirectoryOrCreate - name: plugin-mount-dir hostPath: - path: /var/lib/kubelet/plugins + path: "{{ .KubeletDirPath }}/plugins" type: Directory - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: "{{ .KubeletDirPath }}/plugins_registry/" type: Directory - name: pods-mount-dir hostPath: - path: /var/lib/kubelet/pods + path: "{{ .KubeletDirPath }}/pods" type: Directory - name: host-dev hostPath: diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 63a582781..a0bd3c581 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -181,7 +181,9 @@ spec: # value: "quay.io/k8scsi/csi-snapshotter:v1.2.0" #- name: ROOK_CSI_ATTACHER_IMAGE # value: "quay.io/k8scsi/csi-attacher:v1.2.0" - + # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. + #- name: ROOK_CSI_KUBELET_DIR_PATH + # value: "/var/lib/kubelet" # The name of the node to pass with the downward API - name: NODE_NAME valueFrom: diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index eb1f5ce43..a2f6f0dc6 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -68,6 +68,8 @@ func init() { operatorCmd.Flags().StringVar(&csi.CephFSProvisionerDepTemplatePath, "csi-cephfs-provisioner-dep-template-path", csi.DefaultCephFSProvisionerDepTemplatePath, "path to ceph-csi cephfs provisioner deployment template") operatorCmd.Flags().BoolVar(&csi.EnableCSIGRPCMetrics, "csi-enable-grpc-metrics", true, "enable grpc metrics in ceph-csi") + operatorCmd.Flags().StringVar(&csi.CSIParam.KubeletDirPath, "csi-kubelet-dir-path", csi.DefaultKubeletDirPath, "kubelet directory path for mounting volumes") + operatorCmd.Flags().BoolVar(&disruption.EnableMachineDisruptionBudget, "enable-machine-disruption-budget", false, "enable fencing controllers") flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index e6f519c8f..44f07f7ee 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -36,6 +36,7 @@ type Param struct { SnapshotterImage string DriverNamePrefix string EnableCSIGRPCMetrics string + KubeletDirPath string } type templateParam struct { @@ -77,6 +78,9 @@ const ( DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v1.2.0" DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.2.0" + // kubelet directory path + DefaultKubeletDirPath = "/var/lib/kubelet" + // template DefaultRBDPluginTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml" DefaultRBDProvisionerSTSTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner-sts.yaml"