From 665172a9eadc07e0e0ba14903bb8081dacfebb55 Mon Sep 17 00:00:00 2001 From: Daniel-Pivonka Date: Mon, 8 Jul 2019 16:23:31 -0400 Subject: [PATCH] ceph: deploy liveness sidecar with csi Signed-off-by: Daniel-Pivonka --- Documentation/ceph-csi-drivers.md | 15 ++++ Documentation/ceph-monitoring.md | 8 ++ .../cephfs/csi-cephfsplugin-provisioner.yaml | 36 +++++++++ .../csi/template/cephfs/csi-cephfsplugin.yaml | 36 +++++++++ .../rbd/csi-rbdplugin-provisioner.yaml | 36 +++++++++ .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 37 +++++++++ .../csi-liveness-service-monitor.yaml | 19 +++++ .../framework/installer/ceph_csi_templates.go | 80 +++++++++++++++++++ 8 files changed, 267 insertions(+) create mode 100644 cluster/examples/kubernetes/ceph/monitoring/csi-liveness-service-monitor.yaml diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/ceph-csi-drivers.md index b0e691133..77007856f 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/ceph-csi-drivers.md @@ -1,3 +1,4 @@ + --- title: Ceph CSI weight: 3200 @@ -27,6 +28,7 @@ stage](https://kubernetes.io/blog/2018/10/09/introducing-volume-snapshot-alpha-f (k8s 1.12+), make sure to enable the `VolumeSnapshotDataSource` feature gate on your Kubernetes cluster API server. + ``` --feature-gates=VolumeSnapshotDataSource=true ``` @@ -119,3 +121,16 @@ kubectl delete -f cluster/examples/kubernetes/ceph/csi/example/rbd/pvc-restore.y kubectl delete -f cluster/examples/kubernetes/ceph/csi/example/rbd/snapshot.yaml kubectl delete -f cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml ``` + +#### Liveness Sidecar +All CSI pods are deployed with a sidecar container that provides a prometheus metric for tracking if the CSI plugin is alive and runnning. +These metrics are meant to be collected by prometheus but can be acceses through a GET request to a specific node ip. +for example `curl -X get http://[pod ip]:[liveness-port][liveness-path] 2>/dev/null | grep csi` +the expected output should be +```bash +[root@worker2 /]# curl -X GET http://10.109.65.142:8080/metrics 2>/dev/null | grep csi +# HELP csi_liveness Liveness Probe +# TYPE csi_liveness gauge +csi_liveness 1 +``` +Check the [monitoring doc](https://github.com/rook/rook.github.io/blob/master/docs/rook/master/ceph-monitoring.md) to see how to integrate CSI liveness into ceph monitoring. diff --git a/Documentation/ceph-monitoring.md b/Documentation/ceph-monitoring.md index f5a57dedb..08b0435e9 100644 --- a/Documentation/ceph-monitoring.md +++ b/Documentation/ceph-monitoring.md @@ -125,3 +125,11 @@ Then the rest of the instructions in the [Prometheus Operator docs](https://gith ### Tectonic Bare Metal Tectonic strongly discourages the `tectonic-system` Prometheus instance to be used outside their intentions, so you need to create a new [Prometheus Operator](https://coreos.com/operators/prometheus/docs/latest/) yourself. After this you only need to create the service monitor as stated above. + +### CSI Liveness + +To integrate CSI Liveness into ceph monitoring we will need to deploy a service and service monitor. +```bash +kubectl create -f csi-liveness-service-monitor.yaml +``` +This will create the service monitor to have promethues monitor CSI diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml index b6dd2d2d9..74f8529de 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml @@ -82,6 +82,26 @@ spec: mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8081" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" volumes: - name: socket-dir hostPath: @@ -106,3 +126,19 @@ spec: emptyDir: { medium: "Memory" } +--- +# This is a service to expose the liveness side car +apiVersion: v1 +kind: Service +metadata: + name: csi-liveness-cephfsplugin-provisioner + labels: + app: csi-liveness +spec: + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8081 + selector: + app: csi-cephfsplugin-provisioner diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index a01cee942..f6bfe0176 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -87,6 +87,26 @@ spec: mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8081" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: plugin-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" volumes: - name: plugin-dir hostPath: @@ -125,3 +145,19 @@ spec: emptyDir: { medium: "Memory" } +--- +# This is a service to expose the liveness side car +apiVersion: v1 +kind: Service +metadata: + name: csi-liveness-cephfsplugin + labels: + app: csi-liveness +spec: + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8081 + selector: + app: csi-cephfsplugin diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml index 024bbe1ba..882bf3b82 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml @@ -101,6 +101,26 @@ spec: mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8080" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" volumes: - name: host-dev hostPath: @@ -128,3 +148,19 @@ spec: emptyDir: { medium: "Memory" } +--- +# This is a service to expose the liveness side car +apiVersion: v1 +kind: Service +metadata: + name: csi-liveness-rbdplugin-provisioner + labels: + app: csi-liveness +spec: + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: csi-rbdplugin-provisioner diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index 4f6326fae..cdbd4c10a 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -11,6 +11,7 @@ spec: metadata: labels: app: csi-rbdplugin + contains: liveness spec: serviceAccount: rook-csi-rbd-plugin-sa hostNetwork: true @@ -89,6 +90,26 @@ spec: mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8080" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: plugin-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" volumes: - name: plugin-dir hostPath: @@ -128,3 +149,19 @@ spec: emptyDir: { medium: "Memory" } +--- +# This is a service to expose the liveness side car +apiVersion: v1 +kind: Service +metadata: + name: csi-liveness-rbdplugin + labels: + app: csi-liveness +spec: + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: csi-rbdplugin diff --git a/cluster/examples/kubernetes/ceph/monitoring/csi-liveness-service-monitor.yaml b/cluster/examples/kubernetes/ceph/monitoring/csi-liveness-service-monitor.yaml new file mode 100644 index 000000000..ed617dc68 --- /dev/null +++ b/cluster/examples/kubernetes/ceph/monitoring/csi-liveness-service-monitor.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: csi-liveness + namespace: rook-ceph + labels: + team: rook +spec: + namespaceSelector: + matchNames: + - default + selector: + matchLabels: + app: csi-liveness + endpoints: + - port: http-metrics1 + path: /metrics + interval: 5s diff --git a/tests/framework/installer/ceph_csi_templates.go b/tests/framework/installer/ceph_csi_templates.go index 7f5fb6275..ca4dfb9e9 100644 --- a/tests/framework/installer/ceph_csi_templates.go +++ b/tests/framework/installer/ceph_csi_templates.go @@ -123,6 +123,26 @@ const ( mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8080" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" volumes: - name: host-dev hostPath: @@ -244,6 +264,26 @@ const ( mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8080" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: plugin-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" volumes: - name: plugin-dir hostPath: @@ -369,6 +409,26 @@ const ( mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8081" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" volumes: - name: socket-dir hostPath: @@ -484,6 +544,26 @@ const ( mountPath: /etc/ceph-csi-config/ - name: keys-tmp-dir mountPath: /tmp/csi/keys + - name: liveness-prometheus + image: {{ .CSIPluginImage }} + args: + - "--type=liveness" + - "--endpoint=$(CSI_ENDPOINT)" + - "--livenessport=8081" + - "--livenesspath=/metrics" + - "--polltime=60s" + - "--timeout=3s" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: plugin-dir + mountPath: /csi + imagePullPolicy: "IfNotPresent" volumes: - name: plugin-dir hostPath: