diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index 67a4d3945..29ded5c34 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -52,6 +52,9 @@ If this value is empty, each pod will get an ephemeral directory to store their * `externalMgrPrometheusPort`: external prometheus manager module port. See [external cluster configuration](./external-cluster/external-cluster.md) for more details. * `port`: The internal prometheus manager module port where the prometheus mgr module listens. The port may need to be configured when host networking is enabled. * `interval`: The interval for the prometheus module to to scrape targets. + * `exporter`: Ceph exporter metrics config. + * `perfCountersPrioLimit`: Specifies which performance counters are exported. Corresponds to `--prio-limit` Ceph exporter flag. `0` - all counters are exported, default is `5`. + * `statsPeriodSeconds`: Time to wait before sending requests again to exporter server (seconds). Corresponds to `--stats-period` Ceph exporter flag. Default is `5`. * `network`: For the network settings for the cluster, refer to the [network configuration settings](#network-configuration-settings) * `mon`: contains mon related options [mon settings](#mon-settings) For more details on the mons and when to choose a number other than `3`, see the [mon health doc](../../Storage-Configuration/Advanced/ceph-mon-health.md). diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 662c8575c..ebd043d22 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -3686,6 +3686,45 @@ map[string]int +

CephExporterSpec +

+

+(Appears on:MonitoringSpec) +

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+perfCountersPrioLimit
+ +int64 + +
+

Only performance counters greater than or equal to this option are fetched

+
+statsPeriodSeconds
+ +int64 + +
+

Time to wait before sending requests again to exporter server (seconds)

+

CephFilesystemStatus

@@ -8527,6 +8566,20 @@ Kubernetes meta/v1.Duration

Interval determines prometheus scrape interval

+ + +exporter
+ + +CephExporterSpec + + + + +(Optional) +

Ceph exporter configuration

+ +

MultiClusterServiceSpec diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 767e08498..0092ad799 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -2341,6 +2341,20 @@ spec: Enabled determines whether to create the prometheus rules for the ceph cluster. If true, the prometheus types must exist or the creation will fail. Default is false. type: boolean + exporter: + description: Ceph exporter configuration + properties: + perfCountersPrioLimit: + default: 5 + description: Only performance counters greater than or equal to this option are fetched + format: int64 + type: integer + statsPeriodSeconds: + default: 5 + description: Time to wait before sending requests again to exporter server (seconds) + format: int64 + type: integer + type: object externalMgrEndpoints: description: ExternalMgrEndpoints points to an existing Ceph prometheus exporter endpoint items: diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index 118603403..a01f99f39 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -85,6 +85,15 @@ spec: # Whether to disable the metrics reported by Ceph. If false, the prometheus mgr module and Ceph exporter are enabled. # If true, the prometheus mgr module and Ceph exporter are both disabled. Default is false. metricsDisabled: false + # Ceph exporter metrics config. + exporter: + # Specifies which performance counters are exported. + # Corresponds to --prio-limit Ceph exporter flag + # 0 - all counters are exported + perfCountersPrioLimit: 5 + # Time to wait before sending requests again to exporter server (seconds) + # Corresponds to --stats-period Ceph exporter flag + statsPeriodSeconds: 5 network: connections: # Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 9f7a3d225..aa8759c17 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -2339,6 +2339,20 @@ spec: Enabled determines whether to create the prometheus rules for the ceph cluster. If true, the prometheus types must exist or the creation will fail. Default is false. type: boolean + exporter: + description: Ceph exporter configuration + properties: + perfCountersPrioLimit: + default: 5 + description: Only performance counters greater than or equal to this option are fetched + format: int64 + type: integer + statsPeriodSeconds: + default: 5 + description: Time to wait before sending requests again to exporter server (seconds) + format: int64 + type: integer + type: object externalMgrEndpoints: description: ExternalMgrEndpoints points to an existing Ceph prometheus exporter endpoint items: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index c817ac41a..d06c20081 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -406,6 +406,20 @@ type MonitoringSpec struct { // Interval determines prometheus scrape interval // +optional Interval *metav1.Duration `json:"interval,omitempty"` + + // Ceph exporter configuration + // +optional + Exporter *CephExporterSpec `json:"exporter,omitempty"` +} + +type CephExporterSpec struct { + // Only performance counters greater than or equal to this option are fetched + // +kubebuilder:default=5 + PerfCountersPrioLimit int64 `json:"perfCountersPrioLimit,omitempty"` + + // Time to wait before sending requests again to exporter server (seconds) + // +kubebuilder:default=5 + StatsPeriodSeconds int64 `json:"statsPeriodSeconds,omitempty"` } // ClusterStatus represents the status of a Ceph cluster diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go index 070bee756..dec1809eb 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter.go @@ -42,8 +42,8 @@ const ( monitoringPath = "/etc/ceph-monitoring/" serviceMonitorFile = "exporter-service-monitor.yaml" sockDir = "/run/ceph" - perfCountersPrioLimit = "5" - statsPeriod = "5" + defaultPrioLimit = "5" + defaultStatsPeriod = "5" DefaultMetricsPort uint16 = 9926 exporterServiceMetricName = "ceph-exporter-http-metrics" exporterKeyringUsername = "client.ceph-exporter" @@ -179,10 +179,15 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion exporterEnvVar := generateExporterEnvVar() envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), exporterEnvVar) + prioLimit, statsPeriod := defaultPrioLimit, defaultStatsPeriod + if cephCluster.Spec.Monitoring.Exporter != nil { + prioLimit = strconv.Itoa(int(cephCluster.Spec.Monitoring.Exporter.PerfCountersPrioLimit)) + statsPeriod = strconv.Itoa(int(cephCluster.Spec.Monitoring.Exporter.StatsPeriodSeconds)) + } args := []string{ "--sock-dir", sockDir, "--port", strconv.Itoa(int(DefaultMetricsPort)), - "--prio-limit", perfCountersPrioLimit, + "--prio-limit", prioLimit, "--stats-period", statsPeriod, } diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter_test.go b/pkg/operator/ceph/cluster/nodedaemon/exporter_test.go index 2a6adc90c..17ab9132d 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter_test.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter_test.go @@ -121,6 +121,26 @@ func TestCreateOrUpdateCephExporter(t *testing.T) { assert.Equal(t, tolerations, podSpec.Spec.Tolerations) assert.Equal(t, true, podSpec.Spec.HostNetwork) assert.Equal(t, "test-priority-class", podSpec.Spec.PriorityClassName) + + t.Run("exporter config", func(t *testing.T) { + cephCluster.Spec.Monitoring.Exporter = &cephv1.CephExporterSpec{ + PerfCountersPrioLimit: 3, + StatsPeriodSeconds: 7, + } + res, err := r.createOrUpdateCephExporter(node, tolerations, cephCluster, cephVersion) + assert.NoError(t, err) + assert.Equal(t, controllerutil.OperationResult("updated"), res) + + err = r.client.Get(ctx, types.NamespacedName{Namespace: "rook-ceph", Name: name}, &deploy) + assert.NoError(t, err) + + podSpec := deploy.Spec.Template + args := podSpec.Spec.Containers[0].Args + assert.Equal(t, "--prio-limit", args[4]) + assert.Equal(t, "3", args[5]) + assert.Equal(t, "--stats-period", args[6]) + assert.Equal(t, "7", args[7]) + }) } func TestCephExporterBindAddress(t *testing.T) {