From f54099df0cc20186d6ec7af0aa12f84cb36362b4 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 27 Sep 2019 10:31:05 +0530 Subject: [PATCH] CSI: Make metrics and liveness port configurable provided ENV variables to configure the grpc and liveness metrics port for both cephfs and rbd CSI drivers. Signed-off-by: Madhu Rajanna --- .../rook-ceph/templates/deployment.yaml | 16 +++++++++ cluster/charts/rook-ceph/values.yaml | 4 +++ .../csi-cephfsplugin-provisioner-dep.yaml | 4 +-- .../csi-cephfsplugin-provisioner-sts.yaml | 4 +-- .../template/cephfs/csi-cephfsplugin-svc.yaml | 4 +-- .../csi/template/cephfs/csi-cephfsplugin.yaml | 4 +-- .../rbd/csi-rbdplugin-provisioner-dep.yaml | 4 +-- .../rbd/csi-rbdplugin-provisioner-sts.yaml | 4 +-- .../csi/template/rbd/csi-rbdplugin-svc.yaml | 4 +-- .../ceph/csi/template/rbd/csi-rbdplugin.yaml | 4 +-- .../kubernetes/ceph/operator-openshift.yaml | 10 ++++++ .../examples/kubernetes/ceph/operator.yaml | 10 ++++++ pkg/operator/ceph/csi/spec.go | 33 ++++++++++++++----- pkg/operator/ceph/csi/util.go | 19 +++++++++++ pkg/operator/ceph/csi/util_test.go | 29 ++++++++++++++++ 15 files changed, 129 insertions(+), 24 deletions(-) diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 21e0ddd53..a7bf5b266 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -150,6 +150,22 @@ spec: - name: CSI_PLUGIN_NODE_AFFINITY value: {{ .Values.csi.pluginNodeAffinity }} {{- end }} +{{- if .Values.csi.cephfsGrpcMetricsPort }} + - name: CSI_CEPHFS_GRPC_METRICS_PORT + value: {{ .Values.csi.cephfsGrpcMetricsPort | quote }} +{{- end }} +{{- if .Values.csi.cephfsLivenessMetricsPort }} + - name: CSI_CEPHFS_LIVENESS_METRICS_PORT + value: {{ .Values.csi.cephfsLivenessMetricsPort | quote }} +{{- end }} +{{- if .Values.csi.cephfsrbdGrpcMetricsPort }} + - name: CSI_RBD_GRPC_METRICS_PORT + value: {{ .Values.csi.rbdGrpcMetricsPort | quote }} +{{- end }} +{{- if .Values.csi.rbdLivenessMetricsPort }} + - name: CSI_RBD_LIVENESS_METRICS_PORT + value: {{ .Values.csi.rbdLivenessMetricsPort | quote }} +{{- end }} {{- end }} - name: ROOK_ENABLE_FLEX_DRIVER value: "{{ .Values.enableFlexDriver }}" diff --git a/cluster/charts/rook-ceph/values.yaml b/cluster/charts/rook-ceph/values.yaml index 74c36146e..5a74163cd 100644 --- a/cluster/charts/rook-ceph/values.yaml +++ b/cluster/charts/rook-ceph/values.yaml @@ -73,6 +73,10 @@ csi: # operator: Exists # effect: NoSchedule # pluginNodeAffinity: key1=value1,value2; key2=value3 + #cephfsGrpcMetricsPort: 9091 + #cephfsLivenessMetricsPort: 9081 + #rbdGrpcMetricsPort: 9090 + #rbdLivenessMetricsPort: 9080 #kubeletDirPath: /var/lib/kubelet #cephcsi: #image: quay.io/cephcsi/cephcsi:v1.2.1 diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index fdba6063d..27988be9e 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -64,7 +64,7 @@ spec: - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" - - "--metricsport=9091" + - "--metricsport={{ .CephFSGRPCMetricsPort }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: @@ -102,7 +102,7 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport=9081" + - "--metricsport={{ .CephFSLivenessMetricsPort }}" - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml index 9bef3f6d2..374835e7d 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-sts.yaml @@ -59,7 +59,7 @@ spec: - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" - - "--metricsport=9091" + - "--metricsport={{ .CephFSGRPCMetricsPort }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: @@ -97,7 +97,7 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport=9081" + - "--metricsport={{ .CephFSLivenessMetricsPort }}" - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml index 1f62f775a..2eb4b036c 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml @@ -11,10 +11,10 @@ spec: - name: csi-http-metrics port: 8080 protocol: TCP - targetPort: 9081 + targetPort: {{ .CephFSLivenessMetricsPort }} - name: csi-grpc-metrics port: 8081 protocol: TCP - targetPort: 9081 + targetPort: {{ .CephFSGRPCMetricsPort }} selector: contains: csi-cephfsplugin-metrics diff --git a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index e6b2f168f..10b609898 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -56,7 +56,7 @@ spec: - "--metadatastorage=k8s_configmap" - "--mountcachedir=/mount-cache-dir" - "--pidlimit=-1" - - "--metricsport=9091" + - "--metricsport={{ .CephFSGRPCMetricsPort }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: @@ -102,7 +102,7 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport=9081" + - "--metricsport={{ .CephFSLivenessMetricsPort }}" - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 628c76023..e48c6b222 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -80,7 +80,7 @@ spec: - "--controllerserver=true" - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport=9090" + - "--metricsport={{ .RBDGRPCMetricsPort }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: @@ -118,7 +118,7 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport=9080" + - "--metricsport={{ .RBDLivenessMetricsPort }}" - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml index 4bdd700b4..5079eae24 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-sts.yaml @@ -73,7 +73,7 @@ spec: - "--controllerserver=true" - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport=9090" + - "--metricsport={{ .RBDGRPCMetricsPort }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: @@ -111,7 +111,7 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport=9080" + - "--metricsport={{ .RBDLivenessMetricsPort }}" - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml index 666509aea..6c432669c 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml @@ -11,10 +11,10 @@ spec: - name: csi-http-metrics port: 8080 protocol: TCP - targetPort: 9080 + targetPort: {{ .RBDLivenessMetricsPort }} - name: csi-grpc-metrics port: 8081 protocol: TCP - targetPort: 9080 + targetPort: {{ .RBDGRPCMetricsPort }} selector: contains: csi-rbdplugin-metrics diff --git a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml index 591f0f678..8f7b7aed3 100644 --- a/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -56,7 +56,7 @@ spec: - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--containerized=false" - "--pidlimit=-1" - - "--metricsport=9090" + - "--metricsport={{ .RBDGRPCMetricsPort }}" - "--metricspath=/metrics" - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" env: @@ -100,7 +100,7 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--metricsport=9080" + - "--metricsport={{ .RBDLivenessMetricsPort }}" - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 634aed2ba..565271bf3 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -248,6 +248,16 @@ spec: # - effect: NoExecute # key: node-role.kubernetes.io/etcd # operator: Exists + # Configure CSI cephfs grpc and liveness metrics port + #- name: CSI_CEPHFS_GRPC_METRICS_PORT + # value: "9091" + #- name: CSI_CEPHFS_LIVENESS_METRICS_PORT + # value: "9081" + # Configure CSI rbd grpc and liveness metrics port + #- name: CSI_RBD_GRPC_METRICS_PORT + # value: "9090" + #- name: CSI_RBD_LIVENESS_METRICS_PORT + # value: "9080" - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED value: "true" # The name of the node to pass with the downward API diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index 5280e0269..16bf5d95b 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -210,6 +210,16 @@ spec: # - effect: NoExecute # key: node-role.kubernetes.io/etcd # operator: Exists + # Configure CSI cephfs grpc and liveness metrics port + #- name: CSI_CEPHFS_GRPC_METRICS_PORT + # value: "9091" + #- name: CSI_CEPHFS_LIVENESS_METRICS_PORT + # value: "9081" + # Configure CSI rbd grpc and liveness metrics port + #- name: CSI_RBD_GRPC_METRICS_PORT + # value: "9090" + #- name: CSI_RBD_LIVENESS_METRICS_PORT + # value: "9080" # The name of the node to pass with the downward API - name: NODE_NAME valueFrom: diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index df598ea76..0f7df7a95 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -29,14 +29,18 @@ import ( ) type Param struct { - CSIPluginImage string - RegistrarImage string - ProvisionerImage string - AttacherImage string - SnapshotterImage string - DriverNamePrefix string - EnableCSIGRPCMetrics string - KubeletDirPath string + CSIPluginImage string + RegistrarImage string + ProvisionerImage string + AttacherImage string + SnapshotterImage string + DriverNamePrefix string + EnableCSIGRPCMetrics string + KubeletDirPath string + CephFSGRPCMetricsPort uint16 + CephFSLivenessMetricsPort uint16 + RBDGRPCMetricsPort uint16 + RBDLivenessMetricsPort uint16 } type templateParam struct { @@ -101,6 +105,12 @@ const ( DefaultCephFSProvisionerSTSTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-sts.yaml" DefaultCephFSProvisionerDepTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-dep.yaml" DefaultCephFSPluginServiceTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-svc.yaml" + + // grpc metrics and liveness port for cephfs and rbd + DefaultCephFSGRPCMerticsPort uint16 = 9091 + DefaultCephFSLivenessMerticsPort uint16 = 9081 + DefaultRBDGRPCMerticsPort uint16 = 9090 + DefaultRBDLivenessMerticsPort uint16 = 9080 ) func CSIEnabled() bool { @@ -173,6 +183,13 @@ func StartCSIDrivers(namespace string, clientset kubernetes.Interface, ver *vers tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics) + // parse GRPC and Liveness ports + tp.CephFSGRPCMetricsPort = getPortFromENV("CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort) + tp.CephFSLivenessMetricsPort = getPortFromENV("CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) + + tp.RBDGRPCMetricsPort = getPortFromENV("CSI_RBD_GRPC_METRICS_PORT", DefaultRBDGRPCMerticsPort) + tp.RBDLivenessMetricsPort = getPortFromENV("CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort) + if ver.Minor < provDeploymentSuppVersion { deployProvSTS = true } diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 269dbabb6..e3620fcac 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -21,6 +21,8 @@ import ( "fmt" "io/ioutil" "os" + "strconv" + "strings" "text/template" k8sutil "github.com/rook/rook/pkg/operator/k8sutil" @@ -154,3 +156,20 @@ func applyToPodSpec(pod *corev1.PodSpec, n *corev1.NodeAffinity, t []corev1.Tole NodeAffinity: n, } } + +func getPortFromENV(env string, defaultPort uint16) uint16 { + port := os.Getenv(env) + if strings.TrimSpace(port) == "" { + return defaultPort + } + p, err := strconv.ParseUint(port, 10, 64) + if err != nil { + logger.Debugf("failed to parse port value for env %s. using default port %d. %+v", env, defaultPort, err) + return defaultPort + } + if p > 65535 { + logger.Debugf("%s port value is greater than 65535. using default port %d for %s", port, defaultPort, env) + return defaultPort + } + return uint16(p) +} diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go index 26181cd5b..670aab7d7 100644 --- a/pkg/operator/ceph/csi/util_test.go +++ b/pkg/operator/ceph/csi/util_test.go @@ -114,3 +114,32 @@ func TestStatefulSetTemplate(t *testing.T) { _, err = templateToStatefulSet("test-ss", tmp.Name(), tp) assert.Nil(t, err) } + +func Test_getPortFromENV(t *testing.T) { + var key = "TEST_CSI_PORT_ENV" + var defaultPort uint16 = 8000 + // emtpty env variable + port := getPortFromENV(key, defaultPort) + assert.Equal(t, port, defaultPort) + // valid port is set in env + err := os.Setenv(key, "9000") + assert.Nil(t, err) + port = getPortFromENV(key, defaultPort) + assert.Equal(t, port, uint16(9000)) + err = os.Unsetenv(key) + assert.Nil(t, err) + // higher port value is set in env + err = os.Setenv(key, "65536") + assert.Nil(t, err) + port = getPortFromENV(key, defaultPort) + assert.Equal(t, port, defaultPort) + err = os.Unsetenv(key) + assert.Nil(t, err) + // negative port is set in env + err = os.Setenv(key, "-1") + assert.Nil(t, err) + port = getPortFromENV(key, defaultPort) + assert.Equal(t, port, defaultPort) + err = os.Unsetenv(key) + assert.Nil(t, err) +}