feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

CSI: Make metrics and liveness port configurable

Browse Source 
provided ENV variables to configure the grpc
and liveness metrics port for both cephfs and
rbd CSI drivers.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna
2019-09-27 17:31:45 +05:30
parent 45e3ca2a23
commit f54099df0c
15 changed files with 129 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cluster/charts/rook-ceph/templates/deployment.yaml
+16
View File
@@ -150,6 +150,22 @@ spec:
- name: CSI_PLUGIN_NODE_AFFINITY
value: {{ .Values.csi.pluginNodeAffinity }}
{{- end }}
{{- if .Values.csi.cephfsGrpcMetricsPort }}
- name: CSI_CEPHFS_GRPC_METRICS_PORT
value: {{ .Values.csi.cephfsGrpcMetricsPort | quote }}
{{- end }}
{{- if .Values.csi.cephfsLivenessMetricsPort }}
- name: CSI_CEPHFS_LIVENESS_METRICS_PORT
value: {{ .Values.csi.cephfsLivenessMetricsPort | quote }}
{{- end }}
{{- if .Values.csi.cephfsrbdGrpcMetricsPort }}
- name: CSI_RBD_GRPC_METRICS_PORT
value: {{ .Values.csi.rbdGrpcMetricsPort | quote }}
{{- end }}
{{- if .Values.csi.rbdLivenessMetricsPort }}
- name: CSI_RBD_LIVENESS_METRICS_PORT
value: {{ .Values.csi.rbdLivenessMetricsPort | quote }}
{{- end }}
{{- end }}
- name: ROOK_ENABLE_FLEX_DRIVER
value: "{{ .Values.enableFlexDriver }}"
cluster/charts/rook-ceph/values.yaml
+4
View File
@@ -73,6 +73,10 @@ csi:
# operator: Exists
# effect: NoSchedule
# pluginNodeAffinity: key1=value1,value2; key2=value3
#cephfsGrpcMetricsPort: 9091
#cephfsLivenessMetricsPort: 9081
#rbdGrpcMetricsPort: 9090
#rbdLivenessMetricsPort: 9080
#kubeletDirPath: /var/lib/kubelet
#cephcsi:
#image: quay.io/cephcsi/cephcsi:v1.2.1
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
+2 -2
View File
@@ -64,7 +64,7 @@ spec:
- "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
- "--metadatastorage=k8s_configmap"
- "--pidlimit=-1"
- "--metricsport=9091"
- "--metricsport={{ .CephFSGRPCMetricsPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
@@ -102,7 +102,7 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=9081"
- "--metricsport={{ .CephFSLivenessMetricsPort }}"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml
+2 -2
View File
@@ -59,7 +59,7 @@ spec:
- "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
- "--metadatastorage=k8s_configmap"
- "--pidlimit=-1"
- "--metricsport=9091"
- "--metricsport={{ .CephFSGRPCMetricsPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
@@ -97,7 +97,7 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=9081"
- "--metricsport={{ .CephFSLivenessMetricsPort }}"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml
+2 -2
View File
@@ -11,10 +11,10 @@ spec:
- name: csi-http-metrics
port: 8080
protocol: TCP
targetPort: 9081
targetPort: {{ .CephFSLivenessMetricsPort }}
- name: csi-grpc-metrics
port: 8081
protocol: TCP
targetPort: 9081
targetPort: {{ .CephFSGRPCMetricsPort }}
selector:
contains: csi-cephfsplugin-metrics
cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
+2 -2
View File
@@ -56,7 +56,7 @@ spec:
- "--metadatastorage=k8s_configmap"
- "--mountcachedir=/mount-cache-dir"
- "--pidlimit=-1"
- "--metricsport=9091"
- "--metricsport={{ .CephFSGRPCMetricsPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
@@ -102,7 +102,7 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=9081"
- "--metricsport={{ .CephFSLivenessMetricsPort }}"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
+2 -2
View File
@@ -80,7 +80,7 @@ spec:
- "--controllerserver=true"
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
- "--pidlimit=-1"
- "--metricsport=9090"
- "--metricsport={{ .RBDGRPCMetricsPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
@@ -118,7 +118,7 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=9080"
- "--metricsport={{ .RBDLivenessMetricsPort }}"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml
+2 -2
View File
@@ -73,7 +73,7 @@ spec:
- "--controllerserver=true"
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
- "--pidlimit=-1"
- "--metricsport=9090"
- "--metricsport={{ .RBDGRPCMetricsPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
@@ -111,7 +111,7 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=9080"
- "--metricsport={{ .RBDLivenessMetricsPort }}"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml
+2 -2
View File
@@ -11,10 +11,10 @@ spec:
- name: csi-http-metrics
port: 8080
protocol: TCP
targetPort: 9080
targetPort: {{ .RBDLivenessMetricsPort }}
- name: csi-grpc-metrics
port: 8081
protocol: TCP
targetPort: 9080
targetPort: {{ .RBDGRPCMetricsPort }}
selector:
contains: csi-rbdplugin-metrics
cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml
+2 -2
View File
@@ -56,7 +56,7 @@ spec:
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
- "--containerized=false"
- "--pidlimit=-1"
- "--metricsport=9090"
- "--metricsport={{ .RBDGRPCMetricsPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
env:
@@ -100,7 +100,7 @@ spec:
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=9080"
- "--metricsport={{ .RBDLivenessMetricsPort }}"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
cluster/examples/kubernetes/ceph/operator-openshift.yaml
+10
View File
@@ -248,6 +248,16 @@ spec:
# - effect: NoExecute
# key: node-role.kubernetes.io/etcd
# operator: Exists
# Configure CSI cephfs grpc and liveness metrics port
#- name: CSI_CEPHFS_GRPC_METRICS_PORT
# value: "9091"
#- name: CSI_CEPHFS_LIVENESS_METRICS_PORT
# value: "9081"
# Configure CSI rbd grpc and liveness metrics port
#- name: CSI_RBD_GRPC_METRICS_PORT
# value: "9090"
#- name: CSI_RBD_LIVENESS_METRICS_PORT
# value: "9080"
- name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
value: "true"
# The name of the node to pass with the downward API
cluster/examples/kubernetes/ceph/operator.yaml
+10
View File
@@ -210,6 +210,16 @@ spec:
# - effect: NoExecute
# key: node-role.kubernetes.io/etcd
# operator: Exists
# Configure CSI cephfs grpc and liveness metrics port
#- name: CSI_CEPHFS_GRPC_METRICS_PORT
# value: "9091"
#- name: CSI_CEPHFS_LIVENESS_METRICS_PORT
# value: "9081"
# Configure CSI rbd grpc and liveness metrics port
#- name: CSI_RBD_GRPC_METRICS_PORT
# value: "9090"
#- name: CSI_RBD_LIVENESS_METRICS_PORT
# value: "9080"
# The name of the node to pass with the downward API
- name: NODE_NAME
valueFrom:
pkg/operator/ceph/csi/spec.go
+25 -8
View File
@@ -29,14 +29,18 @@ import (
)
type Param struct {
CSIPluginImage string
RegistrarImage string
ProvisionerImage string
AttacherImage string
SnapshotterImage string
DriverNamePrefix string
EnableCSIGRPCMetrics string
KubeletDirPath string
CSIPluginImage string
RegistrarImage string
ProvisionerImage string
AttacherImage string
SnapshotterImage string
DriverNamePrefix string
EnableCSIGRPCMetrics string
KubeletDirPath string
CephFSGRPCMetricsPort uint16
CephFSLivenessMetricsPort uint16
RBDGRPCMetricsPort uint16
RBDLivenessMetricsPort uint16
}
type templateParam struct {
@@ -101,6 +105,12 @@ const (
DefaultCephFSProvisionerSTSTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-sts.yaml"
DefaultCephFSProvisionerDepTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-dep.yaml"
DefaultCephFSPluginServiceTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-svc.yaml"
// grpc metrics and liveness port for cephfs and rbd
DefaultCephFSGRPCMerticsPort uint16 = 9091
DefaultCephFSLivenessMerticsPort uint16 = 9081
DefaultRBDGRPCMerticsPort uint16 = 9090
DefaultRBDLivenessMerticsPort uint16 = 9080
)
func CSIEnabled() bool {
@@ -173,6 +183,13 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers
tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics)
// parse GRPC and Liveness ports
tp.CephFSGRPCMetricsPort = getPortFromENV("CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort)
tp.CephFSLivenessMetricsPort = getPortFromENV("CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort)
tp.RBDGRPCMetricsPort = getPortFromENV("CSI_RBD_GRPC_METRICS_PORT", DefaultRBDGRPCMerticsPort)
tp.RBDLivenessMetricsPort = getPortFromENV("CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort)
if ver.Minor < provDeploymentSuppVersion {
deployProvSTS = true
}
pkg/operator/ceph/csi/util.go
+19
View File
@@ -21,6 +21,8 @@ import (
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"
"text/template"
k8sutil "github.com/rook/rook/pkg/operator/k8sutil"
@@ -154,3 +156,20 @@ func applyToPodSpec(pod *corev1.PodSpec, n *corev1.NodeAffinity, t []corev1.Tole
NodeAffinity: n,
}
}
func getPortFromENV(env string, defaultPort uint16) uint16 {
port := os.Getenv(env)
if strings.TrimSpace(port) == "" {
return defaultPort
}
p, err := strconv.ParseUint(port, 10, 64)
if err != nil {
logger.Debugf("failed to parse port value for env %s. using default port %d. %+v", env, defaultPort, err)
return defaultPort
}
if p > 65535 {
logger.Debugf("%s port value is greater than 65535. using default port %d for %s", port, defaultPort, env)
return defaultPort
}
return uint16(p)
}
pkg/operator/ceph/csi/util_test.go
+29
View File
@@ -114,3 +114,32 @@ func TestStatefulSetTemplate(t *testing.T) {
_, err = templateToStatefulSet("test-ss", tmp.Name(), tp)
assert.Nil(t, err)
}
func Test_getPortFromENV(t *testing.T) {
var key = "TEST_CSI_PORT_ENV"
var defaultPort uint16 = 8000
// emtpty env variable
port := getPortFromENV(key, defaultPort)
assert.Equal(t, port, defaultPort)
// valid port is set in env
err := os.Setenv(key, "9000")
assert.Nil(t, err)
port = getPortFromENV(key, defaultPort)
assert.Equal(t, port, uint16(9000))
err = os.Unsetenv(key)
assert.Nil(t, err)
// higher port value is set in env
err = os.Setenv(key, "65536")
assert.Nil(t, err)
port = getPortFromENV(key, defaultPort)
assert.Equal(t, port, defaultPort)
err = os.Unsetenv(key)
assert.Nil(t, err)
// negative port is set in env
err = os.Setenv(key, "-1")
assert.Nil(t, err)
port = getPortFromENV(key, defaultPort)
assert.Equal(t, port, defaultPort)
err = os.Unsetenv(key)
assert.Nil(t, err)
}