diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md index 21637187f..31a874764 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/ceph-upgrade.md @@ -270,6 +270,15 @@ below, which you should change to match where your images are located. You can also remove the `ROOK_CSI_CEPHFS_IMAGE` and `ROOK_CSI_RBD_IMAGE` `env` variables that are no longer used in Rook. +If you have configured the kubelet to use other than `/var/lib/kubelet` please +add below to the operator `env` variables. + +```yaml +env: + - name: ROOK_CSI_KUBELET_DIR_PATH + value: "/kubelet/path" +``` + At the same time you edit the CSI driver settings, go ahead and update the operator deployment image: ```yaml image: rook/ceph:v1.1.0 diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 801be12c9..5a5f95b59 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -101,6 +101,12 @@ spec: - name: ROOK_CSI_ENABLE_CEPHFS value: {{ .Values.csi.enableCephfsDriver | quote }} {{- end }} +{{- if .Values.csi }} +{{- if .Values.csi.kubeletDirPath }} + - name: ROOK_CSI_KUBELET_DIR_PATH + value: {{ .Values.csi.kubeletDirPath | quote }} +{{- end }} +{{- end }} {{- if .Values.csi }} - name: ROOK_CSI_ENABLE_GRPC_METRICS value: {{ .Values.csi.enableGrpcMetrics | quote }} diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 9b12e3019..bee02e908 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -59,6 +59,7 @@ csi: enableRbdDriver: true enableCephfsDriver: true enableGrpcMetrics: true + #kubeletDirPath: /var/lib/kubelet #cephcsi: #image: quay.io/cephcsi/cephcsi:v1.2.0 #registrar: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index 74eb24125..fdba6063d 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -120,7 +120,7 @@ spec: volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com" type: DirectoryOrCreate - name: host-sys hostPath: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml index 4882a41c9..9bef3f6d2 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml @@ -115,7 +115,7 @@ spec: volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com" type: DirectoryOrCreate - name: host-sys hostPath: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 15c4d72e2..e6b2f168f 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -24,7 +24,7 @@ spec: args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/csi.sock" + - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/csi.sock" lifecycle: preStop: exec: @@ -79,10 +79,10 @@ spec: - name: plugin-dir mountPath: /csi - name: csi-plugins-dir - mountPath: /var/lib/kubelet/plugins + mountPath: "{{ .KubeletDirPath }}/plugins" mountPropagation: "Bidirectional" - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods + mountPath: "{{ .KubeletDirPath }}/pods" mountPropagation: "Bidirectional" - name: host-sys mountPath: /sys @@ -120,19 +120,19 @@ spec: volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/ + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/" type: DirectoryOrCreate - name: csi-plugins-dir hostPath: - path: /var/lib/kubelet/plugins + path: "{{ .KubeletDirPath }}/plugins" type: Directory - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: "{{ .KubeletDirPath }}/plugins_registry/" type: Directory - name: pods-mount-dir hostPath: - path: /var/lib/kubelet/pods + path: "{{ .KubeletDirPath }}/pods" type: Directory - name: host-sys hostPath: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml index be3b2c4e0..83cdb64da 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml @@ -146,7 +146,7 @@ spec: path: /lib/modules - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com" type: DirectoryOrCreate - name: ceph-csi-config configMap: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index 42f6554fb..932c2b183 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -25,7 +25,7 @@ spec: args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com/csi.sock" + - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com/csi.sock" lifecycle: preStop: exec: @@ -81,10 +81,10 @@ spec: - name: plugin-dir mountPath: /csi - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods + mountPath: "{{ .KubeletDirPath }}/pods" mountPropagation: "Bidirectional" - name: plugin-mount-dir - mountPath: /var/lib/kubelet/plugins + mountPath: "{{ .KubeletDirPath }}/plugins" mountPropagation: "Bidirectional" - mountPath: /dev name: host-dev @@ -122,19 +122,19 @@ spec: volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com" type: DirectoryOrCreate - name: plugin-mount-dir hostPath: - path: /var/lib/kubelet/plugins + path: "{{ .KubeletDirPath }}/plugins" type: Directory - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: "{{ .KubeletDirPath }}/plugins_registry/" type: Directory - name: pods-mount-dir hostPath: - path: /var/lib/kubelet/pods + path: "{{ .KubeletDirPath }}/pods" type: Directory - name: host-dev hostPath: diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 63a582781..a0bd3c581 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -181,7 +181,9 @@ spec: # value: "quay.io/k8scsi/csi-snapshotter:v1.2.0" #- name: ROOK_CSI_ATTACHER_IMAGE # value: "quay.io/k8scsi/csi-attacher:v1.2.0" - + # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. + #- name: ROOK_CSI_KUBELET_DIR_PATH + # value: "/var/lib/kubelet" # The name of the node to pass with the downward API - name: NODE_NAME valueFrom: diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index eb1f5ce43..a2f6f0dc6 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -68,6 +68,8 @@ func init() { operatorCmd.Flags().StringVar(&csi.CephFSProvisionerDepTemplatePath, "csi-cephfs-provisioner-dep-template-path", csi.DefaultCephFSProvisionerDepTemplatePath, "path to ceph-csi cephfs provisioner deployment template") operatorCmd.Flags().BoolVar(&csi.EnableCSIGRPCMetrics, "csi-enable-grpc-metrics", true, "enable grpc metrics in ceph-csi") + operatorCmd.Flags().StringVar(&csi.CSIParam.KubeletDirPath, "csi-kubelet-dir-path", csi.DefaultKubeletDirPath, "kubelet directory path for mounting volumes") + operatorCmd.Flags().BoolVar(&disruption.EnableMachineDisruptionBudget, "enable-machine-disruption-budget", false, "enable fencing controllers") flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index e6f519c8f..44f07f7ee 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -36,6 +36,7 @@ type Param struct { SnapshotterImage string DriverNamePrefix string EnableCSIGRPCMetrics string + KubeletDirPath string } type templateParam struct { @@ -77,6 +78,9 @@ const ( DefaultAttacherImage = "quay.io/k8scsi/csi-attacher:v1.2.0" DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.2.0" + // kubelet directory path + DefaultKubeletDirPath = "/var/lib/kubelet" + // template DefaultRBDPluginTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml" DefaultRBDProvisionerSTSTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner-sts.yaml"