feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

Implement grpc metrics for cephcsi

Browse Source 
grpc metrics is made option in rook
this can be disabled/enabled with ROOK_CSI_ENABLE_GRPC_METRCS
variable in operator.yaml
this will help admin who want to check the grpc metrics
of the ceph-csi plugins

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna
2019-08-30 15:01:45 +05:30
parent 8d5d9b9117
commit a8589af337
18 changed files with 108 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/ceph-csi-drivers.md
+2 -1
View File
@@ -135,4 +135,5 @@ curl -X GET http://10.109.65.142:9080/metrics 2>/dev/null | grep csi
csi_liveness 1
```
Check the [monitoring doc](ceph-monitoring.md) to see how to integrate CSI liveness into ceph monitoring.
Check the [monitoring doc](ceph-monitoring.md) to see how to integrate CSI
liveness and grpc metrics into ceph monitoring.
Documentation/ceph-monitoring.md
+6 -3
View File
@@ -86,7 +86,7 @@ To enable prometheus alerts,
first, create the RBAC rules to enable monitoring,
```BASH
kubectl create -f cluster/examples/kubernetes/ceph/monitoring/rbac.yaml
```
```
then, make following changes to `cluster.yaml` and deploy.
```YAML
monitoring:
@@ -128,8 +128,11 @@ After this you only need to create the service monitor as stated above.
### CSI Liveness
To integrate CSI Liveness into ceph monitoring we will need to deploy a service and service monitor.
To integrate CSI liveness and grpc into ceph monitoring we will need to deploy
a service and service monitor.
```bash
kubectl create -f csi-liveness-service-monitor.yaml
kubectl create -f csi-metrics-service-monitor.yaml
```
This will create the service monitor to have promethues monitor CSI
cluster/charts/rook-ceph/templates/deployment.yaml
+4
View File
@@ -105,6 +105,10 @@ spec:
- name: ROOK_CSI_ENABLE_CEPHFS
value: "{{ .Values.enableCsiCephfsDriver }}"
{{- end }}
{{- if .Values.enableCsiGrpcMetrics }}
- name: ROOK_CSI_ENABLE_GRPC_METRICS
value: "{{ .Values.enableCsiGrpcMetrics }}"
{{- end }}
{{- if .Values.enableFlexDriver }}
- name: ROOK_ENABLE_FLEX_DRIVER
value: "{{ .Values.enableFlexDriver }}"
cluster/charts/rook-ceph/values.yaml
+1
View File
@@ -56,6 +56,7 @@ pspEnable: true
## Settings for whether to disable the drivers or other daemons if they are not needed
enableCsiRbdDriver: true
enableCsiCephfsDriver: true
enableCsiGrpcMetrics: true
enableFlexDriver: true
enableDiscoveryDaemon: true
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
+9 -2
View File
@@ -62,7 +62,14 @@ spec:
- "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
- "--metadatastorage=k8s_configmap"
- "--pidlimit=-1"
- "--metricsport=9091"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
@@ -93,8 +100,8 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=9081"
- "--livenesspath=/metrics"
- "--metricsport=9081"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml
+9 -2
View File
@@ -57,7 +57,14 @@ spec:
- "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
- "--metadatastorage=k8s_configmap"
- "--pidlimit=-1"
- "--metricsport=9091"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
@@ -88,8 +95,8 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=9081"
- "--livenesspath=/metrics"
- "--metricsport=9081"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml
+7 -3
View File
@@ -1,16 +1,20 @@
---
# This is a service to expose the liveness side car
# This is a service to expose the liveness and grpc metrics
apiVersion: v1
kind: Service
metadata:
name: csi-liveness-cephfsplugin-provisioner
name: csi-cephfsplugin-metrics
labels:
app: csi-liveness
app: csi-metrics
spec:
ports:
- name: csi-http-metrics
port: 8080
protocol: TCP
targetPort: 9081
- name: csi-grpc-metrics
port: 8081
protocol: TCP
targetPort: 9081
selector:
contains: csi-cephfsplugin-metrics
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
+9 -2
View File
@@ -55,7 +55,14 @@ spec:
- "--metadatastorage=k8s_configmap"
- "--mountcachedir=/mount-cache-dir"
- "--pidlimit=-1"
- "--metricsport=9091"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
@@ -94,8 +101,8 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=9081"
- "--livenesspath=/metrics"
- "--metricsport=9081"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
+9 -2
View File
@@ -79,7 +79,14 @@ spec:
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
- "--containerized=true"
- "--pidlimit=-1"
- "--metricsport=9090"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: HOST_ROOTFS
value: "/rootfs"
- name: NODE_ID
@@ -114,8 +121,8 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=9080"
- "--livenesspath=/metrics"
- "--metricsport=9080"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml
+9 -2
View File
@@ -72,7 +72,14 @@ spec:
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
- "--containerized=true"
- "--pidlimit=-1"
- "--metricsport=9090"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: HOST_ROOTFS
value: "/rootfs"
- name: NODE_ID
@@ -107,8 +114,8 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=9080"
- "--livenesspath=/metrics"
- "--metricsport=9080"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml
+7 -3
View File
@@ -1,16 +1,20 @@
---
# This is a service to expose the liveness side car
# This is a service to expose the liveness and grpc metrics
apiVersion: v1
kind: Service
metadata:
name: csi-liveness-rbdplugin-provisioner
name: csi-rbdplugin-metrics
labels:
app: csi-liveness
app: csi-metrics
spec:
ports:
- name: csi-http-metrics
port: 8080
protocol: TCP
targetPort: 9080
- name: csi-grpc-metrics
port: 8081
protocol: TCP
targetPort: 9080
selector:
contains: csi-rbdplugin-metrics
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml
+9 -2
View File
@@ -55,7 +55,14 @@ spec:
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
- "--containerized=true"
- "--pidlimit=-1"
- "--metricsport=9090"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: HOST_ROOTFS
value: "/rootfs"
- name: NODE_ID
@@ -96,8 +103,8 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=9080"
- "--livenesspath=/metrics"
- "--metricsport=9080"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
cluster/examples/kubernetes/ceph/monitoring/csi-liveness-service-monitor.yaml → cluster/examples/kubernetes/ceph/monitoring/csi-metrics-service-monitor.yaml
+6 -2
View File
@@ -2,7 +2,7 @@
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: csi-liveness
name: csi-metrics
namespace: rook-ceph
labels:
team: rook
@@ -12,8 +12,12 @@ spec:
- rook-ceph
selector:
matchLabels:
app: csi-liveness
app: csi-metrics
endpoints:
- port: csi-http-metrics
path: /metrics
interval: 5s
# comment csi-grpc-metrics realated information if csi grpc metrics is not enabled
- port: csi-grpc-metrics
path: /metrics
interval: 5s
cluster/examples/kubernetes/ceph/operator-openshift.yaml
+2
View File
@@ -214,6 +214,8 @@ spec:
value: "true"
- name: ROOK_CSI_ENABLE_RBD
value: "true"
- name: ROOK_CSI_ENABLE_GRPC_METRICS
value: "true"
- name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
value: "true"
cluster/examples/kubernetes/ceph/operator.yaml
+2
View File
@@ -155,6 +155,8 @@ spec:
value: "true"
- name: ROOK_CSI_ENABLE_RBD
value: "true"
- name: ROOK_CSI_ENABLE_GRPC_METRICS
value: "true"
# The default version of CSI supported by Rook will be started. To change the version
# of the CSI driver to something other than what is officially supported, change
# these images to the desired release of the CSI driver.
cmd/rook/ceph/operator.go
+2
View File
@@ -65,6 +65,8 @@ func init() {
operatorCmd.Flags().StringVar(&csi.CephFSPluginTemplatePath, "csi-cephfs-plugin-template-path", csi.DefaultCephFSPluginTemplatePath, "path to ceph-csi cephfs plugin template")
operatorCmd.Flags().StringVar(&csi.CephFSProvisionerSTSTemplatePath, "csi-cephfs-provisioner-sts-template-path", csi.DefaultCephFSProvisionerSTSTemplatePath, "path to ceph-csi cephfs provisioner statefulset template")
operatorCmd.Flags().StringVar(&csi.CephFSProvisionerDepTemplatePath, "csi-cephfs-provisioner-dep-template-path", csi.DefaultCephFSProvisionerDepTemplatePath, "path to ceph-csi cephfs provisioner deployment template")
//csi grpc flag
operatorCmd.Flags().BoolVar(&csi.EnableCSIGRPCMetrics, "csi-enable-grpc-metrics", true, "enable grpc metrics in ceph-csi")
flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix)
flags.SetLoggingFlags(operatorCmd.Flags())
pkg/operator/ceph/csi/spec.go
+13 -8
View File
@@ -29,12 +29,13 @@ import (
)
type Param struct {
CSIPluginImage string
RegistrarImage string
ProvisionerImage string
AttacherImage string
SnapshotterImage string
DriverNamePrefix string
CSIPluginImage string
RegistrarImage string
ProvisionerImage string
AttacherImage string
SnapshotterImage string
DriverNamePrefix string
EnableCSIGRPCMetrics string
}
type templateParam struct {
@@ -46,8 +47,9 @@ type templateParam struct {
var (
CSIParam Param
EnableRBD = false
EnableCephFS = false
EnableRBD = false
EnableCephFS = false
EnableCSIGRPCMetrics = false
// template paths
RBDPluginTemplatePath string
@@ -151,6 +153,9 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers
if tp.DriverNamePrefix == "" {
tp.DriverNamePrefix = fmt.Sprintf("%s.", namespace)
}
tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics)
if ver.Minor < provDeploymentSuppVersion {
deployProvSTS = true
}
tests/framework/installer/ceph_manifests.go
+2
View File
@@ -1303,6 +1303,8 @@ spec:
value: "true"
- name: ROOK_CSI_ENABLE_RBD
value: "true"
- name: ROOK_CSI_ENABLE_GRPC_METRICS
value: "true"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1