diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/ceph-csi-drivers.md index 31b94e0cf..b84afc428 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/ceph-csi-drivers.md @@ -135,4 +135,5 @@ curl -X GET http://10.109.65.142:9080/metrics 2>/dev/null | grep csi csi_liveness 1 ``` -Check the [monitoring doc](ceph-monitoring.md) to see how to integrate CSI liveness into ceph monitoring. +Check the [monitoring doc](ceph-monitoring.md) to see how to integrate CSI +liveness and grpc metrics into ceph monitoring. diff --git a/Documentation/ceph-monitoring.md b/Documentation/ceph-monitoring.md index 08b0435e9..93a010bc6 100644 --- a/Documentation/ceph-monitoring.md +++ b/Documentation/ceph-monitoring.md @@ -86,7 +86,7 @@ To enable prometheus alerts, first, create the RBAC rules to enable monitoring, ```BASH kubectl create -f cluster/examples/kubernetes/ceph/monitoring/rbac.yaml -``` +``` then, make following changes to `cluster.yaml` and deploy. ```YAML monitoring: @@ -128,8 +128,11 @@ After this you only need to create the service monitor as stated above. ### CSI Liveness -To integrate CSI Liveness into ceph monitoring we will need to deploy a service and service monitor. +To integrate CSI liveness and grpc into ceph monitoring we will need to deploy +a service and service monitor. + ```bash -kubectl create -f csi-liveness-service-monitor.yaml +kubectl create -f csi-metrics-service-monitor.yaml ``` + This will create the service monitor to have promethues monitor CSI diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index a4c40ae89..d203b5789 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -105,6 +105,10 @@ spec: - name: ROOK_CSI_ENABLE_CEPHFS value: "{{ .Values.enableCsiCephfsDriver }}" {{- end }} +{{- if .Values.enableCsiGrpcMetrics }} + - name: ROOK_CSI_ENABLE_GRPC_METRICS + value: "{{ .Values.enableCsiGrpcMetrics }}" +{{- end }} {{- if .Values.enableFlexDriver }} - name: ROOK_ENABLE_FLEX_DRIVER value: "{{ .Values.enableFlexDriver }}" diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 95ff13bca..8a0d2af78 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -56,6 +56,7 @@ pspEnable: true ## Settings for whether to disable the drivers or other daemons if they are not needed enableCsiRbdDriver: true enableCsiCephfsDriver: true +enableCsiGrpcMetrics: true enableFlexDriver: true enableDiscoveryDaemon: true diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index aca4f36ab..2c3a0efbf 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -62,7 +62,14 @@ spec: - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" + - "--metricsport=9091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -93,8 +100,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=9081" - - "--livenesspath=/metrics" + - "--metricsport=9081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml index 2ff460872..c99ed92a6 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml @@ -57,7 +57,14 @@ spec: - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" + - "--metricsport=9091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -88,8 +95,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=9081" - - "--livenesspath=/metrics" + - "--metricsport=9081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml index 800ac7f10..1f62f775a 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml @@ -1,16 +1,20 @@ --- -# This is a service to expose the liveness side car +# This is a service to expose the liveness and grpc metrics apiVersion: v1 kind: Service metadata: - name: csi-liveness-cephfsplugin-provisioner + name: csi-cephfsplugin-metrics labels: - app: csi-liveness + app: csi-metrics spec: ports: - name: csi-http-metrics port: 8080 protocol: TCP targetPort: 9081 + - name: csi-grpc-metrics + port: 8081 + protocol: TCP + targetPort: 9081 selector: contains: csi-cephfsplugin-metrics diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 858534ba8..caece2f98 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -55,7 +55,14 @@ spec: - "--metadatastorage=k8s_configmap" - "--mountcachedir=/mount-cache-dir" - "--pidlimit=-1" + - "--metricsport=9091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -94,8 +101,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=9081" - - "--livenesspath=/metrics" + - "--metricsport=9081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 9b7411d8c..a5c92b719 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -79,7 +79,14 @@ spec: - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--containerized=true" - "--pidlimit=-1" + - "--metricsport=9090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: HOST_ROOTFS value: "/rootfs" - name: NODE_ID @@ -114,8 +121,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=9080" - - "--livenesspath=/metrics" + - "--metricsport=9080" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml index 94bf78ff5..2da447fdf 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml @@ -72,7 +72,14 @@ spec: - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--containerized=true" - "--pidlimit=-1" + - "--metricsport=9090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: HOST_ROOTFS value: "/rootfs" - name: NODE_ID @@ -107,8 +114,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=9080" - - "--livenesspath=/metrics" + - "--metricsport=9080" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml index 9d128a113..666509aea 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml @@ -1,16 +1,20 @@ --- -# This is a service to expose the liveness side car +# This is a service to expose the liveness and grpc metrics apiVersion: v1 kind: Service metadata: - name: csi-liveness-rbdplugin-provisioner + name: csi-rbdplugin-metrics labels: - app: csi-liveness + app: csi-metrics spec: ports: - name: csi-http-metrics port: 8080 protocol: TCP targetPort: 9080 + - name: csi-grpc-metrics + port: 8081 + protocol: TCP + targetPort: 9080 selector: contains: csi-rbdplugin-metrics diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index 88433cf5e..78520d3a6 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -55,7 +55,14 @@ spec: - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--containerized=true" - "--pidlimit=-1" + - "--metricsport=9090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: HOST_ROOTFS value: "/rootfs" - name: NODE_ID @@ -96,8 +103,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=9080" - - "--livenesspath=/metrics" + - "--metricsport=9080" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/cluster/examples/kubernetes/ceph/monitoring/csi-liveness-service-monitor.yaml b/cluster/examples/kubernetes/ceph/monitoring/csi-metrics-service-monitor.yaml similarity index 59% rename from cluster/examples/kubernetes/ceph/monitoring/csi-liveness-service-monitor.yaml rename to cluster/examples/kubernetes/ceph/monitoring/csi-metrics-service-monitor.yaml index 56a85b70a..ba43fbd2c 100644 --- a/cluster/examples/kubernetes/ceph/monitoring/csi-liveness-service-monitor.yaml +++ b/cluster/examples/kubernetes/ceph/monitoring/csi-metrics-service-monitor.yaml @@ -2,7 +2,7 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: csi-liveness + name: csi-metrics namespace: rook-ceph labels: team: rook @@ -12,8 +12,12 @@ spec: - rook-ceph selector: matchLabels: - app: csi-liveness + app: csi-metrics endpoints: - port: csi-http-metrics path: /metrics interval: 5s + # comment csi-grpc-metrics realated information if csi grpc metrics is not enabled + - port: csi-grpc-metrics + path: /metrics + interval: 5s diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 11dbd8eb7..f53d34978 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -214,6 +214,8 @@ spec: value: "true" - name: ROOK_CSI_ENABLE_RBD value: "true" + - name: ROOK_CSI_ENABLE_GRPC_METRICS + value: "true" - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED value: "true" diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 9f8d78858..a2068282b 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -155,6 +155,8 @@ spec: value: "true" - name: ROOK_CSI_ENABLE_RBD value: "true" + - name: ROOK_CSI_ENABLE_GRPC_METRICS + value: "true" # The default version of CSI supported by Rook will be started. To change the version # of the CSI driver to something other than what is officially supported, change # these images to the desired release of the CSI driver. diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index 81a5d3112..667dcb758 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -65,6 +65,8 @@ func init() { operatorCmd.Flags().StringVar(&csi.CephFSPluginTemplatePath, "csi-cephfs-plugin-template-path", csi.DefaultCephFSPluginTemplatePath, "path to ceph-csi cephfs plugin template") operatorCmd.Flags().StringVar(&csi.CephFSProvisionerSTSTemplatePath, "csi-cephfs-provisioner-sts-template-path", csi.DefaultCephFSProvisionerSTSTemplatePath, "path to ceph-csi cephfs provisioner statefulset template") operatorCmd.Flags().StringVar(&csi.CephFSProvisionerDepTemplatePath, "csi-cephfs-provisioner-dep-template-path", csi.DefaultCephFSProvisionerDepTemplatePath, "path to ceph-csi cephfs provisioner deployment template") + //csi grpc flag + operatorCmd.Flags().BoolVar(&csi.EnableCSIGRPCMetrics, "csi-enable-grpc-metrics", true, "enable grpc metrics in ceph-csi") flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix) flags.SetLoggingFlags(operatorCmd.Flags()) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 1d54e172d..04a02f6f4 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -29,12 +29,13 @@ import ( ) type Param struct { - CSIPluginImage string - RegistrarImage string - ProvisionerImage string - AttacherImage string - SnapshotterImage string - DriverNamePrefix string + CSIPluginImage string + RegistrarImage string + ProvisionerImage string + AttacherImage string + SnapshotterImage string + DriverNamePrefix string + EnableCSIGRPCMetrics string } type templateParam struct { @@ -46,8 +47,9 @@ type templateParam struct { var ( CSIParam Param - EnableRBD = false - EnableCephFS = false + EnableRBD = false + EnableCephFS = false + EnableCSIGRPCMetrics = false // template paths RBDPluginTemplatePath string @@ -151,6 +153,9 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers if tp.DriverNamePrefix == "" { tp.DriverNamePrefix = fmt.Sprintf("%s.", namespace) } + + tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics) + if ver.Minor < provDeploymentSuppVersion { deployProvSTS = true } diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 3eee64a50..75c371daf 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -1303,6 +1303,8 @@ spec: value: "true" - name: ROOK_CSI_ENABLE_RBD value: "true" + - name: ROOK_CSI_ENABLE_GRPC_METRICS + value: "true" --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1