diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 560f18950..8453766f0 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -107,6 +107,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port. | `9090` | | `csi.csiAddonsPort` | CSI Addons server port. | `9070` | | `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port. | `8080` | +| `csi.enableLiveness` | Enable Ceph CSI Liveness sidecar deployment. | `false` | | `csi.forceCephFSKernelClient` | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs. | `true` | | `csi.kubeletDirPath` | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) | `/var/lib/kubelet` | | `csi.cephcsi.image` | Ceph CSI image. | `quay.io/cephcsi/cephcsi:v3.6.2` | diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 05480567b..ec5a8184e 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -159,6 +159,9 @@ data: {{- if .Values.csi.cephfsLivenessMetricsPort }} CSI_CEPHFS_LIVENESS_METRICS_PORT: {{ .Values.csi.cephfsLivenessMetricsPort | quote }} {{- end }} +{{- if .Values.csi.enableLiveness }} + CSI_ENABLE_LIVENESS: {{ .Values.csi.enableLiveness | quote }} +{{- end }} {{- if .Values.csi.rbdGrpcMetricsPort }} CSI_RBD_GRPC_METRICS_PORT: {{ .Values.csi.rbdGrpcMetricsPort | quote }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index d71b2a5f7..02762181c 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -336,6 +336,8 @@ csi: # operator: Exists # effect: NoSchedule # pluginNodeAffinity: key1=value1,value2; key2=value3 + # Set to true to enable Ceph CSI liveness container. + enableLiveness: false #cephfsGrpcMetricsPort: 9091 #cephfsLivenessMetricsPort: 9081 #rbdGrpcMetricsPort: 9090 diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 8a4900e87..a14c039f5 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -489,6 +489,8 @@ data: # cpu: 500m # Configure CSI Ceph FS grpc and liveness metrics port + # Set to true to enable Ceph CSI liveness container. + CSI_ENABLE_LIVENESS: "false" # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" # Configure CSI RBD grpc and liveness metrics port diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 60b8ef95e..911a7b1fd 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -412,6 +412,8 @@ data: # cpu: 500m # Configure CSI Ceph FS grpc and liveness metrics port + # Set to true to enable Ceph CSI liveness container. + CSI_ENABLE_LIVENESS: "false" # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" # Configure CSI RBD grpc and liveness metrics port diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 0efdc446e..b735c77d5 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -131,6 +131,11 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { return errors.Wrap(err, "error getting CSI RBD liveness metrics port.") } + CSIParam.EnableLiveness, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_LIVENESS", "false")) + if err != nil { + return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_LIVENESS'") + } + // default value `system-node-critical` is the highest available priority CSIParam.PluginPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_PRIORITY_CLASSNAME", "") diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index f685e1ef4..5763be922 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -68,6 +68,7 @@ type Param struct { MountCustomCephConf bool EnableOIDCTokenProjection bool EnableCSIEncryption bool + EnableLiveness bool LogLevel uint8 CephFSGRPCMetricsPort uint16 CephFSLivenessMetricsPort uint16 @@ -320,11 +321,14 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI return errors.Wrap(err, "failed to load rbd provisioner deployment template") } - rbdService, err = templateToService("rbd-service", RBDPluginServiceTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load rbd plugin service template") + // Create service if either liveness or GRPC metrics are enabled. + if CSIParam.EnableLiveness || EnableCSIGRPCMetrics { + rbdService, err = templateToService("rbd-service", RBDPluginServiceTemplatePath, tp) + if err != nil { + return errors.Wrap(err, "failed to load rbd plugin service template") + } + rbdService.Namespace = r.opConfig.OperatorNamespace } - rbdService.Namespace = r.opConfig.OperatorNamespace enabledDrivers = append(enabledDrivers, driverDetails{ name: RBDDriverShortName, fullName: RBDDriverName, @@ -344,12 +348,14 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if err != nil { return errors.Wrap(err, "failed to load rbd provisioner deployment template") } - - cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp) - if err != nil { - return errors.Wrap(err, "failed to load cephfs plugin service template") + // Create service if either liveness or GRPC metrics are enabled. + if CSIParam.EnableLiveness || EnableCSIGRPCMetrics { + cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp) + if err != nil { + return errors.Wrap(err, "failed to load cephfs plugin service template") + } + cephfsService.Namespace = r.opConfig.OperatorNamespace } - cephfsService.Namespace = r.opConfig.OperatorNamespace enabledDrivers = append(enabledDrivers, driverDetails{ name: CephFSDriverShortName, fullName: CephFSDriverName, diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index aedcca03a..4a726442f 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -149,6 +149,7 @@ spec: mountPath: /etc/ceph/ceph.conf subPath: ceph.conf {{ end }} + {{ if .EnableLiveness }} - name: liveness-prometheus image: {{ .CSIPluginImage }} args: @@ -169,6 +170,7 @@ spec: - name: socket-dir mountPath: /csi imagePullPolicy: "IfNotPresent" + {{ end }} volumes: - name: socket-dir emptyDir: { diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index b446da630..73a726d97 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -115,6 +115,7 @@ spec: mountPath: /etc/ceph/ceph.conf subPath: ceph.conf {{ end }} + {{ if .EnableLiveness }} - name: liveness-prometheus securityContext: privileged: true @@ -137,6 +138,7 @@ spec: - name: plugin-dir mountPath: /csi imagePullPolicy: "IfNotPresent" + {{ end }} volumes: - name: plugin-dir hostPath: diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 15a2d1274..eea5539c5 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -247,6 +247,7 @@ spec: - name: rook-ceph-csi-kms-config mountPath: /etc/ceph-csi-encryption-kms-config/ {{ end }} + {{ if .EnableLiveness }} - name: liveness-prometheus image: {{ .CSIPluginImage }} args: @@ -267,6 +268,7 @@ spec: - name: socket-dir mountPath: /csi imagePullPolicy: "IfNotPresent" + {{ end }} volumes: - name: host-dev hostPath: diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index abe597bf1..c387abba2 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -177,6 +177,7 @@ spec: - name: plugin-dir mountPath: /csi {{ end }} + {{ if .EnableLiveness }} - name: liveness-prometheus securityContext: privileged: true @@ -199,6 +200,7 @@ spec: - name: plugin-dir mountPath: /csi imagePullPolicy: "IfNotPresent" + {{ end }} volumes: - name: plugin-dir hostPath: