From 187bea6701b5bccebbcdb8a5c0dbefde6962fb9e Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Thu, 28 Jul 2022 13:00:34 +0530 Subject: [PATCH] csi: disable liveness sidecar by default A Liveness sidecar container that is deployed with the CSI pods will be helpful to check csi driver is properly responding or not. and there is no liveness or readiness kubernetes probe added for this one. Providing an option to enable this sidecar deployment and disabling it by default. fixes #8783 Signed-off-by: Madhu Rajanna --- Documentation/Helm-Charts/operator-chart.md | 1 + .../charts/rook-ceph/templates/configmap.yaml | 3 +++ deploy/charts/rook-ceph/values.yaml | 2 ++ deploy/examples/operator-openshift.yaml | 2 ++ deploy/examples/operator.yaml | 2 ++ pkg/operator/ceph/csi/csi.go | 5 ++++ pkg/operator/ceph/csi/spec.go | 24 ++++++++++++------- .../csi-cephfsplugin-provisioner-dep.yaml | 2 ++ .../csi/template/cephfs/csi-cephfsplugin.yaml | 2 ++ .../rbd/csi-rbdplugin-provisioner-dep.yaml | 2 ++ .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 2 ++ 11 files changed, 38 insertions(+), 9 deletions(-) diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 560f18950..8453766f0 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -107,6 +107,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port. | `9090` | | `csi.csiAddonsPort` | CSI Addons server port. | `9070` | | `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port. | `8080` | +| `csi.enableLiveness` | Enable Ceph CSI Liveness sidecar deployment. | `false` | | `csi.forceCephFSKernelClient` | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs. | `true` | | `csi.kubeletDirPath` | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) | `/var/lib/kubelet` | | `csi.cephcsi.image` | Ceph CSI image. | `quay.io/cephcsi/cephcsi:v3.6.2` | diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 05480567b..ec5a8184e 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -159,6 +159,9 @@ data: {{- if .Values.csi.cephfsLivenessMetricsPort }} CSI_CEPHFS_LIVENESS_METRICS_PORT: {{ .Values.csi.cephfsLivenessMetricsPort | quote }} {{- end }} +{{- if .Values.csi.enableLiveness }} + CSI_ENABLE_LIVENESS: {{ .Values.csi.enableLiveness | quote }} +{{- end }} {{- if .Values.csi.rbdGrpcMetricsPort }} CSI_RBD_GRPC_METRICS_PORT: {{ .Values.csi.rbdGrpcMetricsPort | quote }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index d71b2a5f7..02762181c 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -336,6 +336,8 @@ csi: # operator: Exists # effect: NoSchedule # pluginNodeAffinity: key1=value1,value2; key2=value3 + # Set to true to enable Ceph CSI liveness container. + enableLiveness: false #cephfsGrpcMetricsPort: 9091 #cephfsLivenessMetricsPort: 9081 #rbdGrpcMetricsPort: 9090 diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 8a4900e87..a14c039f5 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -489,6 +489,8 @@ data: # cpu: 500m # Configure CSI Ceph FS grpc and liveness metrics port + # Set to true to enable Ceph CSI liveness container. + CSI_ENABLE_LIVENESS: "false" # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" # Configure CSI RBD grpc and liveness metrics port diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 60b8ef95e..911a7b1fd 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -412,6 +412,8 @@ data: # cpu: 500m # Configure CSI Ceph FS grpc and liveness metrics port + # Set to true to enable Ceph CSI liveness container. + CSI_ENABLE_LIVENESS: "false" # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" # Configure CSI RBD grpc and liveness metrics port diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 0efdc446e..b735c77d5 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -131,6 +131,11 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { return errors.Wrap(err, "error getting CSI RBD liveness metrics port.") } + CSIParam.EnableLiveness, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_LIVENESS", "false")) + if err != nil { + return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_LIVENESS'") + } + // default value `system-node-critical` is the highest available priority CSIParam.PluginPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_PRIORITY_CLASSNAME", "") diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index f685e1ef4..5763be922 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -68,6 +68,7 @@ type Param struct { MountCustomCephConf bool EnableOIDCTokenProjection bool EnableCSIEncryption bool + EnableLiveness bool LogLevel uint8 CephFSGRPCMetricsPort uint16 CephFSLivenessMetricsPort uint16 @@ -320,11 +321,14 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI return errors.Wrap(err, "failed to load rbd provisioner deployment template") } - rbdService, err = templateToService("rbd-service", RBDPluginServiceTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load rbd plugin service template") + // Create service if either liveness or GRPC metrics are enabled. + if CSIParam.EnableLiveness || EnableCSIGRPCMetrics { + rbdService, err = templateToService("rbd-service", RBDPluginServiceTemplatePath, tp) + if err != nil { + return errors.Wrap(err, "failed to load rbd plugin service template") + } + rbdService.Namespace = r.opConfig.OperatorNamespace } - rbdService.Namespace = r.opConfig.OperatorNamespace enabledDrivers = append(enabledDrivers, driverDetails{ name: RBDDriverShortName, fullName: RBDDriverName, @@ -344,12 +348,14 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if err != nil { return errors.Wrap(err, "failed to load rbd provisioner deployment template") } - - cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load cephfs plugin service template") + // Create service if either liveness or GRPC metrics are enabled. + if CSIParam.EnableLiveness || EnableCSIGRPCMetrics { + cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp) + if err != nil { + return errors.Wrap(err, "failed to load cephfs plugin service template") + } + cephfsService.Namespace = r.opConfig.OperatorNamespace } - cephfsService.Namespace = r.opConfig.OperatorNamespace enabledDrivers = append(enabledDrivers, driverDetails{ name: CephFSDriverShortName, fullName: CephFSDriverName, diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index aedcca03a..4a726442f 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -149,6 +149,7 @@ spec: mountPath: /etc/ceph/ceph.conf subPath: ceph.conf {{ end }} + {{ if .EnableLiveness }} - name: liveness-prometheus image: {{ .CSIPluginImage }} args: @@ -169,6 +170,7 @@ spec: - name: socket-dir mountPath: /csi imagePullPolicy: "IfNotPresent" + {{ end }} volumes: - name: socket-dir emptyDir: { diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index b446da630..73a726d97 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -115,6 +115,7 @@ spec: mountPath: /etc/ceph/ceph.conf subPath: ceph.conf {{ end }} + {{ if .EnableLiveness }} - name: liveness-prometheus securityContext: privileged: true @@ -137,6 +138,7 @@ spec: - name: plugin-dir mountPath: /csi imagePullPolicy: "IfNotPresent" + {{ end }} volumes: - name: plugin-dir hostPath: diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 15a2d1274..eea5539c5 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -247,6 +247,7 @@ spec: - name: rook-ceph-csi-kms-config mountPath: /etc/ceph-csi-encryption-kms-config/ {{ end }} + {{ if .EnableLiveness }} - name: liveness-prometheus image: {{ .CSIPluginImage }} args: @@ -267,6 +268,7 @@ spec: - name: socket-dir mountPath: /csi imagePullPolicy: "IfNotPresent" + {{ end }} volumes: - name: host-dev hostPath: diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index abe597bf1..c387abba2 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -177,6 +177,7 @@ spec: - name: plugin-dir mountPath: /csi {{ end }} + {{ if .EnableLiveness }} - name: liveness-prometheus securityContext: privileged: true @@ -199,6 +200,7 @@ spec: - name: plugin-dir mountPath: /csi imagePullPolicy: "IfNotPresent" + {{ end }} volumes: - name: plugin-dir hostPath: