Make kubelet path configurable in operator

currently CSI templates have hardcoded kubelet
path which will  be used for  mounting PVC,this
will not work if kubelet is configured to use
different path, with this PR  kubelet path
can be configured for CSI.

Fixes: #3921

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

Documentation/ceph-upgrade.md

@@ -270,6 +270,15 @@ below, which you should change to match where your images are located.
 You can also remove the `ROOK_CSI_CEPHFS_IMAGE` and `ROOK_CSI_RBD_IMAGE` `env` variables that are
 no longer used in Rook.
 
+If you have configured the kubelet to use other than `/var/lib/kubelet` please
+add below to the operator `env` variables.
+
+```yaml
+env:
+    - name: ROOK_CSI_KUBELET_DIR_PATH
+      value: "/kubelet/path"
+```
+
 At the same time you edit the CSI driver settings, go ahead and update the operator deployment image:
 ```yaml
   image: rook/ceph:v1.1.0

cluster/charts/rook-ceph/templates/deployment.yaml

@@ -101,6 +101,12 @@ spec:
         - name: ROOK_CSI_ENABLE_CEPHFS
           value: {{ .Values.csi.enableCephfsDriver | quote }}
 {{- end }}
+{{- if .Values.csi }}
+{{- if .Values.csi.kubeletDirPath }}
+        - name: ROOK_CSI_KUBELET_DIR_PATH
+          value: {{ .Values.csi.kubeletDirPath | quote }}
+{{- end }}
+{{- end }}
 {{- if .Values.csi }}
         - name: ROOK_CSI_ENABLE_GRPC_METRICS
           value: {{ .Values.csi.enableGrpcMetrics | quote }}

cluster/charts/rook-ceph/values.yaml

@@ -59,6 +59,7 @@ csi:
   enableRbdDriver: true
   enableCephfsDriver: true
   enableGrpcMetrics: true
+  #kubeletDirPath: /var/lib/kubelet
   #cephcsi:
     #image: quay.io/cephcsi/cephcsi:v1.2.0
   #registrar:

cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml

@@ -120,7 +120,7 @@ spec:
       volumes:
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com
+            path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com"
             type: DirectoryOrCreate
         - name: host-sys
           hostPath:

cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml

@@ -115,7 +115,7 @@ spec:
       volumes:
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com
+            path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com"
             type: DirectoryOrCreate
         - name: host-sys
           hostPath:

cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml

@@ -24,7 +24,7 @@ spec:
           args:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"
-            - "--kubelet-registration-path=/var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/csi.sock"
+            - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/csi.sock"
           lifecycle:
             preStop:
               exec:
@@ -79,10 +79,10 @@ spec:
             - name: plugin-dir
               mountPath: /csi
             - name: csi-plugins-dir
-              mountPath: /var/lib/kubelet/plugins
+              mountPath: "{{ .KubeletDirPath }}/plugins"
               mountPropagation: "Bidirectional"
             - name: pods-mount-dir
-              mountPath: /var/lib/kubelet/pods
+              mountPath: "{{ .KubeletDirPath }}/pods"
               mountPropagation: "Bidirectional"
             - name: host-sys
               mountPath: /sys
@@ -120,19 +120,19 @@ spec:
       volumes:
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/
+            path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}cephfs.csi.ceph.com/"
             type: DirectoryOrCreate
         - name: csi-plugins-dir
           hostPath:
-            path: /var/lib/kubelet/plugins
+            path: "{{ .KubeletDirPath }}/plugins"
             type: Directory
         - name: registration-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: "{{ .KubeletDirPath }}/plugins_registry/"
             type: Directory
         - name: pods-mount-dir
           hostPath:
-            path: /var/lib/kubelet/pods
+            path: "{{ .KubeletDirPath }}/pods"
             type: Directory
         - name: host-sys
           hostPath:

cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml

@@ -146,7 +146,7 @@ spec:
             path: /lib/modules
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com
+            path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com"
             type: DirectoryOrCreate
         - name: ceph-csi-config
           configMap:

cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml

@@ -25,7 +25,7 @@ spec:
           args:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"
-            - "--kubelet-registration-path=/var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com/csi.sock"
+            - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com/csi.sock"
           lifecycle:
             preStop:
               exec:
@@ -81,10 +81,10 @@ spec:
             - name: plugin-dir
               mountPath: /csi
             - name: pods-mount-dir
-              mountPath: /var/lib/kubelet/pods
+              mountPath: "{{ .KubeletDirPath }}/pods"
               mountPropagation: "Bidirectional"
             - name: plugin-mount-dir
-              mountPath: /var/lib/kubelet/plugins
+              mountPath: "{{ .KubeletDirPath }}/plugins"
               mountPropagation: "Bidirectional"
             - mountPath: /dev
               name: host-dev
@@ -122,19 +122,19 @@ spec:
       volumes:
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com
+            path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com"
             type: DirectoryOrCreate
         - name: plugin-mount-dir
           hostPath:
-            path: /var/lib/kubelet/plugins
+            path: "{{ .KubeletDirPath }}/plugins"
             type: Directory
         - name: registration-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: "{{ .KubeletDirPath }}/plugins_registry/"
             type: Directory
         - name: pods-mount-dir
           hostPath:
-            path: /var/lib/kubelet/pods
+            path: "{{ .KubeletDirPath }}/pods"
             type: Directory
         - name: host-dev
           hostPath:

cluster/examples/kubernetes/ceph/operator.yaml

@@ -181,7 +181,9 @@ spec:
         #  value: "quay.io/k8scsi/csi-snapshotter:v1.2.0"
         #- name: ROOK_CSI_ATTACHER_IMAGE
         #  value: "quay.io/k8scsi/csi-attacher:v1.2.0"
-
+        # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path.
+        #- name: ROOK_CSI_KUBELET_DIR_PATH
+        #  value: "/var/lib/kubelet"
         # The name of the node to pass with the downward API
         - name: NODE_NAME
           valueFrom:

cmd/rook/ceph/operator.go

@@ -68,6 +68,8 @@ func init() {
 	operatorCmd.Flags().StringVar(&csi.CephFSProvisionerDepTemplatePath, "csi-cephfs-provisioner-dep-template-path", csi.DefaultCephFSProvisionerDepTemplatePath, "path to ceph-csi cephfs provisioner deployment template")
 	operatorCmd.Flags().BoolVar(&csi.EnableCSIGRPCMetrics, "csi-enable-grpc-metrics", true, "enable grpc metrics in ceph-csi")
 
+	operatorCmd.Flags().StringVar(&csi.CSIParam.KubeletDirPath, "csi-kubelet-dir-path", csi.DefaultKubeletDirPath, "kubelet directory path for mounting volumes")
+
 	operatorCmd.Flags().BoolVar(&disruption.EnableMachineDisruptionBudget, "enable-machine-disruption-budget", false, "enable fencing controllers")
 
 	flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix)

pkg/operator/ceph/csi/spec.go

@@ -36,6 +36,7 @@ type Param struct {
 	SnapshotterImage     string
 	DriverNamePrefix     string
 	EnableCSIGRPCMetrics string
+	KubeletDirPath       string
 }
 
 type templateParam struct {
@@ -77,6 +78,9 @@ const (
 	DefaultAttacherImage    = "quay.io/k8scsi/csi-attacher:v1.2.0"
 	DefaultSnapshotterImage = "quay.io/k8scsi/csi-snapshotter:v1.2.0"
 
+	// kubelet directory path
+	DefaultKubeletDirPath = "/var/lib/kubelet"
+
 	// template
 	DefaultRBDPluginTemplatePath         = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml"
 	DefaultRBDProvisionerSTSTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner-sts.yaml"