From 3edadd13b90af4d2ce70150bf2124063bc8d1fa8 Mon Sep 17 00:00:00 2001 From: Jared Watts Date: Wed, 23 Aug 2017 08:56:14 -0700 Subject: [PATCH] metrics: Include number of slow requests Ported from https://github.com/digitalocean/ceph_exporter/pull/66 --- pkg/ceph/collectors/health.go | 21 +++++++++++++++++++++ pkg/ceph/collectors/health_test.go | 18 ++++++++++++++++++ pkg/ceph/collectors/osd.go | 1 - 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/pkg/ceph/collectors/health.go b/pkg/ceph/collectors/health.go index 85d49ee79..945688df8 100644 --- a/pkg/ceph/collectors/health.go +++ b/pkg/ceph/collectors/health.go @@ -78,6 +78,9 @@ type ClusterHealthCollector struct { // in that state. StuckStalePGs prometheus.Gauge + // SlowRequests depicts no. of total slow requests in the cluster + SlowRequests prometheus.Gauge + // DegradedObjectsCount gives the no. of RADOS objects are constitute the degraded PGs. // This includes object replicas in its count. DegradedObjectsCount prometheus.Gauge @@ -217,6 +220,13 @@ func NewClusterHealthCollector(context *clusterd.Context, clusterName string) *C Help: "No. of stuck stale PGs in the cluster", }, ), + SlowRequests: prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: cephNamespace, + Name: "slow_requests", + Help: "No. of slow requests", + }, + ), DegradedObjectsCount: prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: cephNamespace, @@ -358,6 +368,7 @@ func (c *ClusterHealthCollector) metricsList() []prometheus.Metric { c.StuckUndersizedPGs, c.StalePGs, c.StuckStalePGs, + c.SlowRequests, c.DegradedObjectsCount, c.MisplacedObjectsCount, c.OSDsDown, @@ -407,6 +418,7 @@ func (c *ClusterHealthCollector) collect(stats cephclient.CephStatus) error { stuckUndersizedRegex = regexp.MustCompile(`([\d]+) pgs stuck undersized`) staleRegex = regexp.MustCompile(`([\d]+) pgs stale`) stuckStaleRegex = regexp.MustCompile(`([\d]+) pgs stuck stale`) + slowRequestRegex = regexp.MustCompile(`([\d]+) slow requests are blocked`) degradedObjectsRegex = regexp.MustCompile(`recovery ([\d]+)/([\d]+) objects degraded`) misplacedObjectsRegex = regexp.MustCompile(`recovery ([\d]+)/([\d]+) objects misplaced`) ) @@ -484,6 +496,15 @@ func (c *ClusterHealthCollector) collect(stats cephclient.CephStatus) error { c.StuckStalePGs.Set(float64(v)) } + matched = slowRequestRegex.FindStringSubmatch(m.Summary.Message) + if len(matched) == 2 { + v, err := strconv.Atoi(matched[1]) + if err != nil { + return err + } + c.SlowRequests.Set(float64(v)) + } + matched = degradedObjectsRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 3 { v, err := strconv.Atoi(matched[1]) diff --git a/pkg/ceph/collectors/health_test.go b/pkg/ceph/collectors/health_test.go index b857e0bb2..4bda4616d 100644 --- a/pkg/ceph/collectors/health_test.go +++ b/pkg/ceph/collectors/health_test.go @@ -360,6 +360,24 @@ func TestClusterHealthCollector(t *testing.T) { regexp.MustCompile(`total_pgs 52000`), }, }, + { + input: ` +{ + "health": { + "checks": { + "REQUEST_SLOW": { + "severity": "HEALTH_WARN", + "summary": { + "message": "6 slow requests are blocked > 32 sec" + } + } + } + } +}`, + regexes: []*regexp.Regexp{ + regexp.MustCompile(`slow_requests 6`), + }, + }, } { func() { executor := &exectest.MockExecutor{ diff --git a/pkg/ceph/collectors/osd.go b/pkg/ceph/collectors/osd.go index a5f66648d..821e503c8 100644 --- a/pkg/ceph/collectors/osd.go +++ b/pkg/ceph/collectors/osd.go @@ -289,7 +289,6 @@ func (o *OSDCollector) collect() error { depth, err := node.Depth.Float64() if err != nil { - return err }