From 5658e026cba0950b576b0d01a1f0f3e92fbd8ac2 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 7 Nov 2023 11:02:39 +0100 Subject: [PATCH] csi: remove deprecated grpc metrics code GRPC metrics got deprecated in cephcsi 3.7.0 and the deprecated flags will get removed in the next release. This PR removes the deprecated metrics code which allow us to run with older cephcsi as well. Signed-off-by: Madhu Rajanna --- Documentation/Helm-Charts/operator-chart.md | 3 --- .../Ceph-CSI/ceph-csi-drivers.md | 3 ++- .../Monitoring/ceph-monitoring.md | 2 +- .../charts/rook-ceph/templates/configmap.yaml | 7 ------- .../rook-ceph/templates/servicemonitor.yaml | 5 ----- deploy/charts/rook-ceph/values.yaml | 10 ---------- .../monitoring/csi-metrics-service-monitor.yaml | 4 ---- deploy/examples/operator-openshift.yaml | 7 ++----- deploy/examples/operator.yaml | 7 ++----- pkg/operator/ceph/csi/csi.go | 17 +---------------- pkg/operator/ceph/csi/spec.go | 8 ++------ .../csi-cephfsplugin-provisioner-dep.yaml | 3 --- .../csi/template/cephfs/csi-cephfsplugin.yaml | 3 --- .../rbd/csi-rbdplugin-provisioner-dep.yaml | 3 --- .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 3 --- 15 files changed, 10 insertions(+), 75 deletions(-) diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index e7b5f06eb..a8731f491 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -61,7 +61,6 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` | | `csi.cephFSPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. | `1` | | `csi.cephcsi.image` | Ceph CSI image | `quay.io/cephcsi/cephcsi:v3.9.0` | -| `csi.cephfsGrpcMetricsPort` | CSI CephFS driver GRPC metrics port | `9091` | | `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port | `9081` | | `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Deployments and DaemonSets Pods | `nil` | | `csi.clusterName` | Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster | `nil` | @@ -82,7 +81,6 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.enableCSIHostNetwork` | Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary in some network configurations where the SDN does not provide access to an external cluster or there is significant drop in read/write performance | `true` | | `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver | `true` | | `csi.enableCephfsSnapshotter` | Enable Snapshotter in CephFS provisioner pod | `true` | -| `csi.enableGrpcMetrics` | Enable Ceph CSI GRPC Metrics | `false` | | `csi.enableLiveness` | Enable Ceph CSI Liveness sidecar deployment | `false` | | `csi.enableMetadata` | Enable adding volume metadata on the CephFS subvolumes and RBD images. Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. Hence enable metadata is false by default | `false` | | `csi.enableNFSSnapshotter` | Enable Snapshotter in NFS provisioner pod | `true` | @@ -110,7 +108,6 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.provisionerTolerations` | Array of tolerations in YAML format which will be added to CSI provisioner deployment | `nil` | | `csi.rbdAttachRequired` | Whether to skip any attach operation altogether for RBD PVCs. See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). If set to false it skips the volume attachments and makes the creation of pods using the RBD PVC fast. **WARNING** It's highly discouraged to use this for RWO volumes as it can cause data corruption. csi-addons operations like Reclaimspace and PVC Keyrotation will also not be supported if set to false since we'll have no VolumeAttachments to determine which node the PVC is mounted on. Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. | `true` | | `csi.rbdFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html | `"File"` | -| `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port | `9090` | | `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port | `8080` | | `csi.rbdPluginUpdateStrategy` | CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` | | `csi.rbdPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI RBD plugin daemonset update strategy. | `1` | diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md index 4bab0c1b8..43eed829f 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md @@ -48,7 +48,7 @@ csi_liveness 1 ``` Check the [monitoring doc](../Monitoring/ceph-monitoring.md) to see how to integrate CSI -liveness and grpc metrics into ceph monitoring. +liveness metrics into ceph monitoring. ## Dynamically Expand Volume @@ -253,6 +253,7 @@ Execute the following steps: * Patch the `rook-ceph-operator-config` configmap using the following command. + ```console kubectl patch cm rook-ceph-operator-config -nrook-ceph -p $'data:\n "CSI_ENABLE_READ_AFFINITY": "true"' ``` diff --git a/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md index b5f9b30ba..7cb6e352f 100644 --- a/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md +++ b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md @@ -261,7 +261,7 @@ After this you only need to create the service monitor as stated above. ### CSI Liveness -To integrate CSI liveness and grpc into ceph monitoring we will need to deploy +To integrate CSI liveness into ceph monitoring we will need to deploy a service and service monitor. ```console diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 440bcf854..d23f3a022 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -61,7 +61,6 @@ data: {{- if .Values.csi.kubeletDirPath }} ROOK_CSI_KUBELET_DIR_PATH: {{ .Values.csi.kubeletDirPath | quote }} {{- end }} - ROOK_CSI_ENABLE_GRPC_METRICS: {{ .Values.csi.enableGrpcMetrics | quote }} {{- if .Values.csi.cephcsi }} {{- if .Values.csi.cephcsi.image }} ROOK_CSI_CEPH_IMAGE: {{ .Values.csi.cephcsi.image | quote }} @@ -176,18 +175,12 @@ data: {{- if .Values.csi.nfsPluginNodeAffinity }} CSI_NFS_PLUGIN_NODE_AFFINITY: {{ .Values.csi.nfsPluginNodeAffinity }} {{- end }} -{{- if .Values.csi.cephfsGrpcMetricsPort }} - CSI_CEPHFS_GRPC_METRICS_PORT: {{ .Values.csi.cephfsGrpcMetricsPort | quote }} -{{- end }} {{- if .Values.csi.cephfsLivenessMetricsPort }} CSI_CEPHFS_LIVENESS_METRICS_PORT: {{ .Values.csi.cephfsLivenessMetricsPort | quote }} {{- end }} {{- if .Values.csi.enableLiveness }} CSI_ENABLE_LIVENESS: {{ .Values.csi.enableLiveness | quote }} {{- end }} -{{- if .Values.csi.rbdGrpcMetricsPort }} - CSI_RBD_GRPC_METRICS_PORT: {{ .Values.csi.rbdGrpcMetricsPort | quote }} -{{- end }} {{- if .Values.csi.rbdLivenessMetricsPort }} CSI_RBD_LIVENESS_METRICS_PORT: {{ .Values.csi.rbdLivenessMetricsPort | quote }} {{- end }} diff --git a/deploy/charts/rook-ceph/templates/servicemonitor.yaml b/deploy/charts/rook-ceph/templates/servicemonitor.yaml index a9634d0e6..b25dbe848 100644 --- a/deploy/charts/rook-ceph/templates/servicemonitor.yaml +++ b/deploy/charts/rook-ceph/templates/servicemonitor.yaml @@ -21,9 +21,4 @@ spec: path: /metrics interval: {{ .Values.csi.serviceMonitor.interval }} {{- end }} - {{- if .Values.csi.enableGrpcMetrics }} - - port: csi-grpc-metrics - path: /metrics - interval: {{ .Values.csi.serviceMonitor.interval }} - {{- end }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 638b42c3c..e77e0ff4a 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -82,8 +82,6 @@ csi: enableRbdDriver: true # -- Enable Ceph CSI CephFS driver enableCephfsDriver: true - # -- Enable Ceph CSI GRPC Metrics - enableGrpcMetrics: false # -- Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary # in some network configurations where the SDN does not provide access to an external cluster or # there is significant drop in read/write performance @@ -448,18 +446,10 @@ csi: # -- Enable Ceph CSI Liveness sidecar deployment enableLiveness: false - # -- CSI CephFS driver GRPC metrics port - # @default -- `9091` - cephfsGrpcMetricsPort: - # -- CSI CephFS driver metrics port # @default -- `9081` cephfsLivenessMetricsPort: - # -- Ceph CSI RBD driver GRPC metrics port - # @default -- `9090` - rbdGrpcMetricsPort: - # -- CSI Addons server port # @default -- `9070` csiAddonsPort: diff --git a/deploy/examples/monitoring/csi-metrics-service-monitor.yaml b/deploy/examples/monitoring/csi-metrics-service-monitor.yaml index ea223d53f..f1f2675f9 100644 --- a/deploy/examples/monitoring/csi-metrics-service-monitor.yaml +++ b/deploy/examples/monitoring/csi-metrics-service-monitor.yaml @@ -17,7 +17,3 @@ spec: - port: csi-http-metrics path: /metrics interval: 5s - # comment csi-grpc-metrics related information if csi grpc metrics is not enabled - - port: csi-grpc-metrics - path: /metrics - interval: 5s diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 5c402ebe6..125112d9a 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -121,7 +121,6 @@ data: ROOK_CSI_ENABLE_RBD: "true" # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below. ROOK_CSI_ENABLE_NFS: "false" - ROOK_CSI_ENABLE_GRPC_METRICS: "false" # Set to true to enable Ceph CSI pvc encryption support. CSI_ENABLE_ENCRYPTION: "false" @@ -549,13 +548,11 @@ data: # memory: 1Gi # cpu: 500m - # Configure CSI Ceph FS grpc and liveness metrics port + # Configure CSI CephFS liveness metrics port # Set to true to enable Ceph CSI liveness container. CSI_ENABLE_LIVENESS: "false" - # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" - # Configure CSI RBD grpc and liveness metrics port - # CSI_RBD_GRPC_METRICS_PORT: "9090" + # Configure CSI RBD liveness metrics port # CSI_RBD_LIVENESS_METRICS_PORT: "9080" # CSIADDONS_PORT: "9070" diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 808f7cc77..2d980a5f4 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -35,7 +35,6 @@ data: ROOK_CSI_ENABLE_RBD: "true" # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below. ROOK_CSI_ENABLE_NFS: "false" - ROOK_CSI_ENABLE_GRPC_METRICS: "false" # Set to true to enable Ceph CSI pvc encryption support. CSI_ENABLE_ENCRYPTION: "false" @@ -476,13 +475,11 @@ data: # memory: 1Gi # cpu: 500m - # Configure CSI Ceph FS grpc and liveness metrics port + # Configure CSI CephFS liveness metrics port # Set to true to enable Ceph CSI liveness container. CSI_ENABLE_LIVENESS: "false" - # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" - # Configure CSI RBD grpc and liveness metrics port - # CSI_RBD_GRPC_METRICS_PORT: "9090" + # Configure CSI RBD liveness metrics port # CSI_RBD_LIVENESS_METRICS_PORT: "9080" # CSIADDONS_PORT: "9070" diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index c85364739..49d79423b 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -17,7 +17,6 @@ limitations under the License. package csi import ( - "fmt" "strconv" "strings" "time" @@ -81,11 +80,6 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ALLOW_UNSUPPORTED_VERSION'") } - if EnableCSIGRPCMetrics, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_GRPC_METRICS", "false")); err != nil { - return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_GRPC_METRICS'") - } - CSIParam.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics) - if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_HOST_NETWORK", "true")); err != nil { return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'") } @@ -115,20 +109,11 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { } CSIParam.GRPCTimeout = time.Duration(timeoutSeconds) * time.Second - // parse GRPC and Liveness ports - CSIParam.CephFSGRPCMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort) - if err != nil { - return errors.Wrap(err, "error getting CSI CephFS GRPC metrics port.") - } + // parse Liveness port CSIParam.CephFSLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI CephFS liveness metrics port.") } - - CSIParam.RBDGRPCMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_GRPC_METRICS_PORT", DefaultRBDGRPCMerticsPort) - if err != nil { - return errors.Wrap(err, "error getting CSI RBD GRPC metrics port.") - } CSIParam.CSIAddonsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_PORT", DefaultCSIAddonsPort) if err != nil { return errors.Wrap(err, "failed to get CSI Addons port") diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 94743c037..025933dfb 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -46,7 +46,6 @@ type Param struct { SnapshotterImage string ResizerImage string DriverNamePrefix string - EnableCSIGRPCMetrics string KubeletDirPath string ForceCephFSKernelClient string CephFSKernelMountOptions string @@ -83,9 +82,7 @@ type Param struct { NFSAttachRequired bool LogLevel uint8 SidecarLogLevel uint8 - CephFSGRPCMetricsPort uint16 CephFSLivenessMetricsPort uint16 - RBDGRPCMetricsPort uint16 CSIAddonsPort uint16 RBDLivenessMetricsPort uint16 ProvisionerReplicas int32 @@ -115,7 +112,6 @@ var ( EnableRBD = false EnableCephFS = false EnableNFS = false - EnableCSIGRPCMetrics = false AllowUnsupported = false CustomCSICephConfigExists = false @@ -321,7 +317,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI } // Create service if either liveness or GRPC metrics are enabled. - if CSIParam.EnableLiveness || EnableCSIGRPCMetrics { + if CSIParam.EnableLiveness { rbdService, err = templateToService("rbd-service", RBDPluginServiceTemplatePath, tp) if err != nil { return errors.Wrap(err, "failed to load rbd plugin service template") @@ -348,7 +344,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI return errors.Wrap(err, "failed to load rbd provisioner deployment template") } // Create service if either liveness or GRPC metrics are enabled. - if CSIParam.EnableLiveness || EnableCSIGRPCMetrics { + if CSIParam.EnableLiveness { cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp) if err != nil { return errors.Wrap(err, "failed to load cephfs plugin service template") diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index f2ba703df..14374971a 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -112,10 +112,7 @@ spec: - "--controllerserver=true" - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport={{ .CephFSGRPCMetricsPort }}" - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - - "--metricspath=/metrics" - - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" {{ if .CSIEnableMetadata }} - "--setmetadata={{ .CSIEnableMetadata }}" {{ end }} diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 47fd21a02..1f2b306e4 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -71,10 +71,7 @@ spec: - "--nodeserver=true" - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport={{ .CephFSGRPCMetricsPort }}" - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - - "--metricspath=/metrics" - - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" {{ if .CephFSKernelMountOptions }} - "--kernelmountoptions={{ .CephFSKernelMountOptions }}" {{ end }} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 872a27407..99a710d05 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -181,9 +181,6 @@ spec: - "--controllerserver=true" - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport={{ .RBDGRPCMetricsPort }}" - - "--metricspath=/metrics" - - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" {{ if .EnableCSIAddonsSideCar }} - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)" {{ end }} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index 8c023b2db..95153f402 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -72,9 +72,6 @@ spec: - "--nodeserver=true" - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport={{ .RBDGRPCMetricsPort }}" - - "--metricspath=/metrics" - - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" {{ if .EnableCSIAddonsSideCar }} - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"