feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #3416 from Daniel-Pivonka/csi-liveness

Browse Source 
ceph: deploy liveness sidecar with csi
This commit is contained in:
Travis Nielsen authored and GitHub committed 2019-08-21 07:42:57 -06:00
2 parents 7a2c006c4d 665172a9ea
commit db2b267cb5
8 files changed
+267
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/ceph-csi-drivers.md
+15
View File
@@ -1,3 +1,4 @@
---
title: Ceph CSI
weight: 3200
@@ -27,6 +28,7 @@ stage](https://kubernetes.io/blog/2018/10/09/introducing-volume-snapshot-alpha-f
(k8s 1.12+), make sure to enable the `VolumeSnapshotDataSource` feature gate on
your Kubernetes cluster API server.
```
--feature-gates=VolumeSnapshotDataSource=true
```
@@ -119,3 +121,16 @@ kubectl delete -f cluster/examples/kubernetes/ceph/csi/example/rbd/pvc-restore.y
kubectl delete -f cluster/examples/kubernetes/ceph/csi/example/rbd/snapshot.yaml
kubectl delete -f cluster/examples/kubernetes/ceph/csi/example/rbd/snapshotclass.yaml
```
#### Liveness Sidecar
All CSI pods are deployed with a sidecar container that provides a prometheus metric for tracking if the CSI plugin is alive and runnning.
These metrics are meant to be collected by prometheus but can be acceses through a GET request to a specific node ip.
for example `curl -X get http://[pod ip]:[liveness-port][liveness-path] 2>/dev/null | grep csi`
the expected output should be
```bash
[root@worker2 /]# curl -X GET http://10.109.65.142:8080/metrics 2>/dev/null | grep csi
# HELP csi_liveness Liveness Probe
# TYPE csi_liveness gauge
csi_liveness 1
```
Check the [monitoring doc](https://github.com/rook/rook.github.io/blob/master/docs/rook/master/ceph-monitoring.md) to see how to integrate CSI liveness into ceph monitoring.
Documentation/ceph-monitoring.md
+8
View File
@@ -125,3 +125,11 @@ Then the rest of the instructions in the [Prometheus Operator docs](https://gith
### Tectonic Bare Metal
Tectonic strongly discourages the `tectonic-system` Prometheus instance to be used outside their intentions, so you need to create a new [Prometheus Operator](https://coreos.com/operators/prometheus/docs/latest/) yourself.
After this you only need to create the service monitor as stated above.
### CSI Liveness
To integrate CSI Liveness into ceph monitoring we will need to deploy a service and service monitor.
```bash
kubectl create -f csi-liveness-service-monitor.yaml
```
This will create the service monitor to have promethues monitor CSI
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner.yaml
+36
View File
@@ -82,6 +82,26 @@ spec:
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=8081"
- "--livenesspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: socket-dir
hostPath:
@@ -106,3 +126,19 @@ spec:
emptyDir: {
medium: "Memory"
}
---
# This is a service to expose the liveness side car
apiVersion: v1
kind: Service
metadata:
name: csi-liveness-cephfsplugin-provisioner
labels:
app: csi-liveness
spec:
ports:
- name: http-metrics
port: 8080
protocol: TCP
targetPort: 8081
selector:
app: csi-cephfsplugin-provisioner
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
+36
View File
@@ -87,6 +87,26 @@ spec:
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=8081"
- "--livenesspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: plugin-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: plugin-dir
hostPath:
@@ -125,3 +145,19 @@ spec:
emptyDir: {
medium: "Memory"
}
---
# This is a service to expose the liveness side car
apiVersion: v1
kind: Service
metadata:
name: csi-liveness-cephfsplugin
labels:
app: csi-liveness
spec:
ports:
- name: http-metrics
port: 8080
protocol: TCP
targetPort: 8081
selector:
app: csi-cephfsplugin
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner.yaml
+36
View File
@@ -101,6 +101,26 @@ spec:
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=8080"
- "--livenesspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: host-dev
hostPath:
@@ -128,3 +148,19 @@ spec:
emptyDir: {
medium: "Memory"
}
---
# This is a service to expose the liveness side car
apiVersion: v1
kind: Service
metadata:
name: csi-liveness-rbdplugin-provisioner
labels:
app: csi-liveness
spec:
ports:
- name: http-metrics
port: 8080
protocol: TCP
targetPort: 8080
selector:
app: csi-rbdplugin-provisioner
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml
+37
View File
@@ -11,6 +11,7 @@ spec:
metadata:
labels:
app: csi-rbdplugin
contains: liveness
spec:
serviceAccount: rook-csi-rbd-plugin-sa
hostNetwork: true
@@ -89,6 +90,26 @@ spec:
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=8080"
- "--livenesspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: plugin-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: plugin-dir
hostPath:
@@ -128,3 +149,19 @@ spec:
emptyDir: {
medium: "Memory"
}
---
# This is a service to expose the liveness side car
apiVersion: v1
kind: Service
metadata:
name: csi-liveness-rbdplugin
labels:
app: csi-liveness
spec:
ports:
- name: http-metrics
port: 8080
protocol: TCP
targetPort: 8080
selector:
app: csi-rbdplugin
cluster/examples/kubernetes/ceph/monitoring/csi-liveness-service-monitor.yaml
+19
View File
@@ -0,0 +1,19 @@
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: csi-liveness
namespace: rook-ceph
labels:
team: rook
spec:
namespaceSelector:
matchNames:
- default
selector:
matchLabels:
app: csi-liveness
endpoints:
- port: http-metrics1
path: /metrics
interval: 5s
tests/framework/installer/ceph_csi_templates.go
+80
View File
@@ -123,6 +123,26 @@ const (
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=8080"
- "--livenesspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: host-dev
hostPath:
@@ -244,6 +264,26 @@ const (
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=8080"
- "--livenesspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: plugin-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: plugin-dir
hostPath:
@@ -369,6 +409,26 @@ const (
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=8081"
- "--livenesspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: socket-dir
hostPath:
@@ -484,6 +544,26 @@ const (
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--livenessport=8081"
- "--livenesspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: plugin-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: plugin-dir
hostPath: