exporter: add configurable port for ceph exporter

the ceph exporter port was hardcoded to 9926, preventing
co-location of exporters from different ceph clusters on
the same node. add a port field to cephexporterspec so
users can override it via spec.monitoring.exporter.port.

Signed-off-by: Oded Viner <oviner@redhat.com>
(cherry picked from commit c2a2fc4241)

Documentation/CRDs/specification.md

@@ -4032,6 +4032,18 @@ int64
 </tr>
 <tr>
 <td>
+<code>port</code><br/>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Port is the listening port of the ceph-exporter process. Defaults to 9926.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>hostNetwork</code><br/>
 <em>
 bool

deploy/charts/rook-ceph/templates/resources.yaml

@@ -2929,6 +2929,12 @@ spec:
                           description: Only performance counters greater than or equal to this option are fetched
                           format: int64
                           type: integer
+                        port:
+                          description: Port is the listening port of the ceph-exporter process. Defaults to 9926.
+                          format: int32
+                          maximum: 65535
+                          minimum: 1
+                          type: integer
                         statsPeriodSeconds:
                           default: 5
                           description: Time to wait before sending requests again to exporter server (seconds)

deploy/examples/crds.yaml

@@ -2927,6 +2927,12 @@ spec:
                           description: Only performance counters greater than or equal to this option are fetched
                           format: int64
                           type: integer
+                        port:
+                          description: Port is the listening port of the ceph-exporter process. Defaults to 9926.
+                          format: int32
+                          maximum: 65535
+                          minimum: 1
+                          type: integer
                         statsPeriodSeconds:
                           default: 5
                           description: Time to wait before sending requests again to exporter server (seconds)

pkg/apis/ceph.rook.io/v1/types.go

@@ -508,6 +508,12 @@ type CephExporterSpec struct {
 	// +kubebuilder:default=5
 	StatsPeriodSeconds int64 `json:"statsPeriodSeconds,omitempty"`
 
+	// Port is the listening port of the ceph-exporter process. Defaults to 9926.
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Maximum=65535
+	// +optional
+	Port int32 `json:"port,omitempty"`
+
 	// Whether host networking is enabled for CephExporter. If not set, the network settings from CephCluster.spec.networking will be applied.
 	// +nullable
 	// +optional

pkg/operator/ceph/cluster/nodedaemon/exporter.go

@@ -178,6 +178,13 @@ func getCephExporterChownInitContainer(cephCluster cephv1.CephCluster) corev1.Co
 	)
 }
 
+func getExporterMetricsPort(cephCluster cephv1.CephCluster) int32 {
+	if cephCluster.Spec.Monitoring.Exporter != nil && cephCluster.Spec.Monitoring.Exporter.Port != 0 {
+		return cephCluster.Spec.Monitoring.Exporter.Port
+	}
+	return int32(DefaultMetricsPort)
+}
+
 func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container {
 	cephImage := cephCluster.Spec.CephVersion.Image
 	dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath)
@@ -187,6 +194,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
 	exporterEnvVar := generateExporterEnvVar()
 	envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), exporterEnvVar)
 
+	metricsPort := getExporterMetricsPort(cephCluster)
 	prioLimit, statsPeriod := defaultPrioLimit, defaultStatsPeriod
 	if cephCluster.Spec.Monitoring.Exporter != nil {
 		prioLimit = strconv.Itoa(int(cephCluster.Spec.Monitoring.Exporter.PerfCountersPrioLimit))
@@ -194,7 +202,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
 	}
 	args := []string{
 		"--sock-dir", sockDir,
-		"--port", strconv.Itoa(int(DefaultMetricsPort)),
+		"--port", strconv.Itoa(int(metricsPort)),
 		"--prio-limit", prioLimit,
 		"--stats-period", statsPeriod,
 	}
@@ -206,7 +214,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
 
 	containerPort := corev1.ContainerPort{
 		Name:          "http-metrics",
-		ContainerPort: int32(DefaultMetricsPort),
+		ContainerPort: metricsPort,
 		Protocol:      corev1.ProtocolTCP,
 	}
 
@@ -241,7 +249,7 @@ func MakeCephExporterMetricsService(cephCluster cephv1.CephCluster, servicePortM
 			Ports: []corev1.ServicePort{
 				{
 					Name:     servicePortMetricName,
-					Port:     int32(DefaultMetricsPort),
+					Port:     getExporterMetricsPort(cephCluster),
 					Protocol: corev1.ProtocolTCP,
 				},
 			},
@@ -304,7 +312,7 @@ func applyPrometheusAnnotations(cephCluster cephv1.CephCluster, objectMeta *meta
 	if len(cephv1.GetCephExporterAnnotations(cephCluster.Spec.Annotations)) == 0 {
 		t := cephv1.Annotations{
 			"prometheus.io/scrape": "true",
-			"prometheus.io/port":   strconv.Itoa(int(DefaultMetricsPort)),
+			"prometheus.io/port":   strconv.Itoa(int(getExporterMetricsPort(cephCluster))),
 		}
 
 		t.ApplyToObjectMeta(objectMeta)

pkg/operator/ceph/cluster/nodedaemon/exporter_test.go

@@ -41,6 +41,10 @@ import (
 )
 
 func assertCephExporterArgs(t *testing.T, args []string, ipv6 bool) {
+	assertCephExporterArgsWithPort(t, args, ipv6, "9926")
+}
+
+func assertCephExporterArgsWithPort(t *testing.T, args []string, ipv6 bool, port string) {
 	var expectedArgsLen int
 	if ipv6 {
 		expectedArgsLen = 10
@@ -53,7 +57,7 @@ func assertCephExporterArgs(t *testing.T, args []string, ipv6 bool) {
 	assert.Equal(t, "--sock-dir", args[0])
 	assert.Equal(t, "/run/ceph", args[1])
 	assert.Equal(t, "--port", args[2])
-	assert.Equal(t, "9926", args[3])
+	assert.Equal(t, port, args[3])
 	assert.Equal(t, "--prio-limit", args[4])
 	assert.Equal(t, "5", args[5])
 	assert.Equal(t, "--stats-period", args[6])
@@ -144,6 +148,25 @@ func TestCreateOrUpdateCephExporter(t *testing.T) {
 		assert.Equal(t, "--stats-period", args[6])
 		assert.Equal(t, "7", args[7])
 	})
+
+	t.Run("exporter custom port", func(t *testing.T) {
+		cephCluster.Spec.Monitoring.Exporter = &cephv1.CephExporterSpec{
+			PerfCountersPrioLimit: 5,
+			StatsPeriodSeconds:    5,
+			Port:                  9927,
+		}
+		res, err := r.createOrUpdateCephExporter(node, tolerations, cephCluster, cephVersion)
+		assert.NoError(t, err)
+		assert.Equal(t, controllerutil.OperationResult("updated"), res)
+
+		err = r.client.Get(ctx, types.NamespacedName{Namespace: "rook-ceph", Name: name}, &deploy)
+		assert.NoError(t, err)
+
+		podSpec := deploy.Spec.Template
+		args := podSpec.Spec.Containers[0].Args
+		assertCephExporterArgsWithPort(t, args, false, "9927")
+		assert.Equal(t, int32(9927), podSpec.Spec.Containers[0].Ports[0].ContainerPort)
+	})
 }
 
 func TestCephExporterLogrotateContainer(t *testing.T) {
@@ -408,8 +431,17 @@ func TestServiceSpec(t *testing.T) {
 	assert.NotNil(t, s)
 	assert.Equal(t, "rook-ceph-exporter", s.Name)
 	assert.Equal(t, 1, len(s.Spec.Ports))
+	assert.Equal(t, int32(DefaultMetricsPort), s.Spec.Ports[0].Port)
 	assert.Equal(t, 2, len(s.Labels))
 	assert.Equal(t, 2, len(s.Spec.Selector))
+
+	t.Run("custom exporter port", func(t *testing.T) {
+		cephCluster.Spec.Monitoring.Exporter = &cephv1.CephExporterSpec{Port: 9927}
+		s, err := MakeCephExporterMetricsService(cephCluster, exporterServiceMetricName, scheme.Scheme)
+		assert.NoError(t, err)
+		assert.NotNil(t, s)
+		assert.Equal(t, int32(9927), s.Spec.Ports[0].Port)
+	})
 }
 
 func TestApplyCephExporterLabels(t *testing.T) {