From ed7f757c06c990a5629ceecf6d38532ebfdfd66b Mon Sep 17 00:00:00 2001 From: Jared Watts Date: Fri, 11 Aug 2017 15:50:40 -0700 Subject: [PATCH] monitoring: update metrics collection for new Ceph Luminous formats --- pkg/api/status.go | 2 +- pkg/api/status_test.go | 2 +- pkg/ceph/client/status.go | 28 +- pkg/ceph/client/status_test.go | 24 +- pkg/ceph/collectors/health.go | 275 +++---------------- pkg/ceph/collectors/health_test.go | 423 +++++++++++++++-------------- 6 files changed, 295 insertions(+), 459 deletions(-) diff --git a/pkg/api/status.go b/pkg/api/status.go index 4e50874ae..7e66487d3 100644 --- a/pkg/api/status.go +++ b/pkg/api/status.go @@ -39,7 +39,7 @@ func (h *Handler) GetStatusDetails(w http.ResponseWriter, r *http.Request) { checks[i] = model.StatusSummary{ Name: name, Status: ceph.HealthToModelHealthStatus(check.Severity), - Message: check.Message, + Message: check.Summary.Message, } i++ } diff --git a/pkg/api/status_test.go b/pkg/api/status_test.go index 145ec8cd2..52c6049d6 100644 --- a/pkg/api/status_test.go +++ b/pkg/api/status_test.go @@ -30,7 +30,7 @@ import ( const ( // this JSON was generated from the command: ceph status - cephStatusResponseRaw = `{"fsid":"d64ecb40-3e93-44e8-9957-2faba55c56de","health":{"checks":{"TEST_MSG":{"severity":"HEALTH_WARN","message":"too many PGs per OSD (2048 > max 300)"}},"status":"HEALTH_WARN"},"election_epoch":6,"quorum":[0],"quorum_names":["rook-ceph-mon0"],"monmap":{"epoch":3,"fsid":"d64ecb40-3e93-44e8-9957-2faba55c56de","modified":"2017-08-01 16:50:42.901253","created":"2017-08-01 16:50:30.751733","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"127.0.0.1:6790/0","public_addr":"127.0.0.1:6790/0"}]},"osdmap":{"osdmap":{"epoch":8,"num_osds":10,"num_up_osds":9,"num_in_osds":9,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"created+peering","count": 100},{"state_name": "active+clean","count": 2048}],"num_pgs":2148,"num_pools":0,"num_objects":0,"data_bytes":0,"bytes_used":123,"bytes_avail":234,"bytes_total":345},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":4109,"active_name":"rook-ceph-mgr0","active_addr":"172.17.0.8:6800/13","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}` + cephStatusResponseRaw = `{"fsid":"d64ecb40-3e93-44e8-9957-2faba55c56de","health":{"checks":{"TEST_MSG":{"severity":"HEALTH_WARN","summary":{"message":"too many PGs per OSD (2048 > max 300)"}}},"status":"HEALTH_WARN"},"election_epoch":6,"quorum":[0],"quorum_names":["rook-ceph-mon0"],"monmap":{"epoch":3,"fsid":"d64ecb40-3e93-44e8-9957-2faba55c56de","modified":"2017-08-01 16:50:42.901253","created":"2017-08-01 16:50:30.751733","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"127.0.0.1:6790/0","public_addr":"127.0.0.1:6790/0"}]},"osdmap":{"osdmap":{"epoch":8,"num_osds":10,"num_up_osds":9,"num_in_osds":9,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"created+peering","count": 100},{"state_name": "active+clean","count": 2048}],"num_pgs":2148,"num_pools":0,"num_objects":0,"data_bytes":0,"bytes_used":123,"bytes_avail":234,"bytes_total":345},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":4109,"active_name":"rook-ceph-mgr0","active_addr":"172.17.0.8:6800/13","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}` // this JSON was generated from the command: ceph time-sync-status cephTimeStatusReponseRaw = `{"time_skew_status":{"rook-ceph-mon0":{"skew":0.000000,"latency":0.000000,"health":"HEALTH_OK"}},"timechecks":{"epoch":6,"round":6,"round_status":"finished"}}` ) diff --git a/pkg/ceph/client/status.go b/pkg/ceph/client/status.go index e4c91ed1b..acb8d3ee1 100644 --- a/pkg/ceph/client/status.go +++ b/pkg/ceph/client/status.go @@ -59,7 +59,9 @@ type HealthStatus struct { type CheckMessage struct { Severity string `json:"severity"` - Message string `json:"message"` + Summary struct { + Message string `json:"message"` + } `json:"summary"` } type MonMap struct { @@ -95,13 +97,23 @@ type OsdMap struct { } type PgMap struct { - PgsByState []PgStateEntry `json:"pgs_by_state"` - Version int `json:"version"` - NumPgs int `json:"num_pgs"` - DataBytes uint64 `json:"data_bytes"` - UsedBytes uint64 `json:"bytes_used"` - AvailableBytes uint64 `json:"bytes_avail"` - TotalBytes uint64 `json:"bytes_total"` + PgsByState []PgStateEntry `json:"pgs_by_state"` + Version int `json:"version"` + NumPgs int `json:"num_pgs"` + DataBytes uint64 `json:"data_bytes"` + UsedBytes uint64 `json:"bytes_used"` + AvailableBytes uint64 `json:"bytes_avail"` + TotalBytes uint64 `json:"bytes_total"` + ReadBps uint64 `json:"read_bytes_sec"` + WriteBps uint64 `json:"write_bytes_sec"` + ReadOps uint64 `json:"read_op_per_sec"` + WriteOps uint64 `json:"write_op_per_sec"` + RecoveryBps uint64 `json:"recovering_bytes_per_sec"` + RecoveryObjectsPerSec uint64 `json:"recovering_objects_per_sec"` + RecoveryKeysPerSec uint64 `json:"recovering_keys_per_sec"` + CacheFlushBps uint64 `json:"flush_bytes_sec"` + CacheEvictBps uint64 `json:"evict_bytes_sec"` + CachePromoteBps uint64 `json:"promote_op_per_sec"` } type PgStateEntry struct { diff --git a/pkg/ceph/client/status_test.go b/pkg/ceph/client/status_test.go index 3d2958968..e826b4fdc 100644 --- a/pkg/ceph/client/status_test.go +++ b/pkg/ceph/client/status_test.go @@ -23,8 +23,8 @@ import ( ) const ( - // this JSON was generated from the mon_command "status", ExecuteMonCommand(conn, map[string]interface{}{"prefix": "status"}) - CephStatusResponseRaw = `{"fsid":"3ca36e66-bd6a-49d4-a976-cdac216b78c3","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","message":"1 osds down"},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","message":"1 host (1 osds) down"},"OSD_ROOT_DOWN":{"severity":"HEALTH_WARN","message":"1 root (1 osds) down"},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","message":"Reduced data availability: 100 pgs stale"}},"status":"HEALTH_WARN"},"election_epoch":22,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon1","rook-ceph-mon2"],"monmap":{"epoch":1,"fsid":"3ca36e66-bd6a-49d4-a976-cdac216b78c3","modified":"2017-07-28 02:39:57.490676","created":"2017-07-28 02:39:57.490676","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.0.0.13:6790/0","public_addr":"10.0.0.13:6790/0"},{"rank":1,"name":"rook-ceph-mon1","addr":"10.0.0.102:6790/0","public_addr":"10.0.0.102:6790/0"},{"rank":2,"name":"rook-ceph-mon2","addr":"10.0.0.227:6790/0","public_addr":"10.0.0.227:6790/0"}]},"osdmap":{"osdmap":{"epoch":13,"num_osds":1,"num_up_osds":0,"num_in_osds":1,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":100}],"num_pgs":100,"num_pools":1,"num_objects":0,"data_bytes":0,"bytes_used":3464249344,"bytes_avail":13829283840,"bytes_total":17293533184},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":4112,"active_name":"rook-ceph-mgr0","active_addr":"172.17.0.10:6800/13","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}` + // this JSON was generated from `ceph status -f json`, using Ceph Luminous 12.1.3 + CephStatusResponseRaw = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6790/0","public_addr":"10.3.0.45:6790/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6790/0","public_addr":"10.3.0.249:6790/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6790/0","public_addr":"10.3.0.252:6790/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}` ) func TestStatusMarshal(t *testing.T) { @@ -36,24 +36,24 @@ func TestStatusMarshal(t *testing.T) { assert.Equal(t, "HEALTH_WARN", status.Health.Status) assert.Equal(t, 4, len(status.Health.Checks)) assert.Equal(t, "HEALTH_WARN", status.Health.Checks["OSD_DOWN"].Severity) - assert.Equal(t, "1 osds down", status.Health.Checks["OSD_DOWN"].Message) - assert.Equal(t, "HEALTH_WARN", status.Health.Checks["OSD_ROOT_DOWN"].Severity) - assert.Equal(t, "1 root (1 osds) down", status.Health.Checks["OSD_ROOT_DOWN"].Message) + assert.Equal(t, "1 osds down", status.Health.Checks["OSD_DOWN"].Summary.Message) + assert.Equal(t, "HEALTH_WARN", status.Health.Checks["OSD_HOST_DOWN"].Severity) + assert.Equal(t, "1 host (1 osds) down", status.Health.Checks["OSD_HOST_DOWN"].Summary.Message) // verify some Mon map fields - assert.Equal(t, 1, status.MonMap.Epoch) + assert.Equal(t, 3, status.MonMap.Epoch) assert.Equal(t, "rook-ceph-mon0", status.MonMap.Mons[0].Name) - assert.Equal(t, "10.0.0.13:6790/0", status.MonMap.Mons[0].Address) + assert.Equal(t, "10.3.0.45:6790/0", status.MonMap.Mons[0].Address) // verify some OSD map fields - assert.Equal(t, 1, status.OsdMap.OsdMap.NumOsd) - assert.Equal(t, 0, status.OsdMap.OsdMap.NumUpOsd) + assert.Equal(t, 2, status.OsdMap.OsdMap.NumOsd) + assert.Equal(t, 1, status.OsdMap.OsdMap.NumUpOsd) assert.False(t, status.OsdMap.OsdMap.Full) assert.True(t, status.OsdMap.OsdMap.NearFull) // verify some PG map fields - assert.Equal(t, 100, status.PgMap.NumPgs) - assert.Equal(t, uint64(3464249344), status.PgMap.UsedBytes) - assert.Equal(t, 100, status.PgMap.PgsByState[0].Count) + assert.Equal(t, 200, status.PgMap.NumPgs) + assert.Equal(t, uint64(13611479040), status.PgMap.UsedBytes) + assert.Equal(t, 101, status.PgMap.PgsByState[0].Count) assert.Equal(t, "stale+active+clean", status.PgMap.PgsByState[0].StateName) } diff --git a/pkg/ceph/collectors/health.go b/pkg/ceph/collectors/health.go index ba2487d4b..85d49ee79 100644 --- a/pkg/ceph/collectors/health.go +++ b/pkg/ceph/collectors/health.go @@ -20,12 +20,8 @@ which has the same license. package collectors import ( - "bufio" - "bytes" - "fmt" "regexp" "strconv" - "strings" "github.com/prometheus/client_golang/prometheus" @@ -33,22 +29,6 @@ import ( "github.com/rook/rook/pkg/clusterd" ) -var ( - recoveryIORateRegex = regexp.MustCompile(`(\d+) (\w{2})/s`) - recoveryIOKeysRegex = regexp.MustCompile(`(\d+) keys/s`) - recoveryIOObjectsRegex = regexp.MustCompile(`(\d+) objects/s`) - clientIOReadRegex = regexp.MustCompile(`(\d+) ([kKmMgG][bB])/s rd`) - clientIOWriteRegex = regexp.MustCompile(`(\d+) ([kKmMgG][bB])/s wr`) - clientIOReadOpsRegex = regexp.MustCompile(`(\d+) op/s rd`) - clientIOWriteOpsRegex = regexp.MustCompile(`(\d+) op/s wr`) - cacheFlushRateRegex = regexp.MustCompile(`(\d+) ([kKmMgG][bB])/s flush`) - cacheEvictRateRegex = regexp.MustCompile(`(\d+) ([kKmMgG][bB])/s evict`) - cachePromoteOpsRegex = regexp.MustCompile(`(\d+) op/s promote`) - - // Older versions of Ceph, hammer (v0.94) and below, support this format. - clientIOOpsRegex = regexp.MustCompile(`(\d+) op/s[^ \w]*$`) -) - // ClusterHealthCollector collects information about the health of an overall cluster. // It surfaces changes in the ceph parameters unlike data usage that ClusterUsageCollector // does. @@ -399,11 +379,7 @@ func (c *ClusterHealthCollector) metricsList() []prometheus.Metric { } } -func (c *ClusterHealthCollector) collect() error { - stats, err := cephclient.Status(c.context, c.clusterName) - if err != nil { - return err - } +func (c *ClusterHealthCollector) collect(stats cephclient.CephStatus) error { for _, metric := range c.metricsList() { if gauge, ok := metric.(prometheus.Gauge); ok { @@ -436,7 +412,7 @@ func (c *ClusterHealthCollector) collect() error { ) for _, m := range stats.Health.Checks { - matched := degradedRegex.FindStringSubmatch(m.Message) + matched := degradedRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 2 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -445,7 +421,7 @@ func (c *ClusterHealthCollector) collect() error { c.DegradedPGs.Set(float64(v)) } - matched = stuckDegradedRegex.FindStringSubmatch(m.Message) + matched = stuckDegradedRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 2 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -454,7 +430,7 @@ func (c *ClusterHealthCollector) collect() error { c.StuckDegradedPGs.Set(float64(v)) } - matched = uncleanRegex.FindStringSubmatch(m.Message) + matched = uncleanRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 2 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -463,7 +439,7 @@ func (c *ClusterHealthCollector) collect() error { c.UncleanPGs.Set(float64(v)) } - matched = stuckUncleanRegex.FindStringSubmatch(m.Message) + matched = stuckUncleanRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 2 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -472,7 +448,7 @@ func (c *ClusterHealthCollector) collect() error { c.StuckUncleanPGs.Set(float64(v)) } - matched = undersizedRegex.FindStringSubmatch(m.Message) + matched = undersizedRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 2 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -481,7 +457,7 @@ func (c *ClusterHealthCollector) collect() error { c.UndersizedPGs.Set(float64(v)) } - matched = stuckUndersizedRegex.FindStringSubmatch(m.Message) + matched = stuckUndersizedRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 2 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -490,7 +466,7 @@ func (c *ClusterHealthCollector) collect() error { c.StuckUndersizedPGs.Set(float64(v)) } - matched = staleRegex.FindStringSubmatch(m.Message) + matched = staleRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 2 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -499,7 +475,7 @@ func (c *ClusterHealthCollector) collect() error { c.StalePGs.Set(float64(v)) } - matched = stuckStaleRegex.FindStringSubmatch(m.Message) + matched = stuckStaleRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 2 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -508,7 +484,7 @@ func (c *ClusterHealthCollector) collect() error { c.StuckStalePGs.Set(float64(v)) } - matched = degradedObjectsRegex.FindStringSubmatch(m.Message) + matched = degradedObjectsRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 3 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -517,7 +493,7 @@ func (c *ClusterHealthCollector) collect() error { c.DegradedObjectsCount.Set(float64(v)) } - matched = misplacedObjectsRegex.FindStringSubmatch(m.Message) + matched = misplacedObjectsRegex.FindStringSubmatch(m.Summary.Message) if len(matched) == 3 { v, err := strconv.Atoi(matched[1]) if err != nil { @@ -549,220 +525,51 @@ func (c *ClusterHealthCollector) collect() error { return nil } -func (c *ClusterHealthCollector) collectRecoveryClientIO() error { - buf, err := cephclient.StatusPlain(c.context, c.clusterName) - if err != nil { +func (c *ClusterHealthCollector) collectRecoveryClientIO(stats cephclient.CephStatus) error { + if err := c.collectRecoveryIO(stats); err != nil { return err } - sc := bufio.NewScanner(bytes.NewReader(buf)) - for sc.Scan() { - line := strings.TrimSpace(sc.Text()) - - switch { - case strings.HasPrefix(line, "recovery io"): - if err := c.collectRecoveryIO(line); err != nil { - return err - } - case strings.HasPrefix(line, "client io"): - if err := c.collectClientIO(line); err != nil { - return err - } - case strings.HasPrefix(line, "cache io"): - if err := c.collectCacheIO(line); err != nil { - return err - } - } + if err := c.collectClientIO(stats); err != nil { + return err } + + if err := c.collectCacheIO(stats); err != nil { + return err + } + return nil } -func (c *ClusterHealthCollector) collectClientIO(clientStr string) error { - matched := clientIOReadRegex.FindStringSubmatch(clientStr) - if len(matched) == 3 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } +func (c *ClusterHealthCollector) collectClientIO(stats cephclient.CephStatus) error { + c.ClientIORead.Set(float64(stats.PgMap.ReadBps)) + c.ClientIOWrite.Set(float64(stats.PgMap.WriteBps)) - switch strings.ToLower(matched[2]) { - case "gb": - v = v * 1e9 - case "mb": - v = v * 1e6 - case "kb": - v = v * 1e3 - default: - return fmt.Errorf("can't parse units %q", matched[2]) - } + clientIOReadOps := float64(stats.PgMap.ReadOps) + c.ClientIOReadOps.Set(clientIOReadOps) - c.ClientIORead.Set(float64(v)) - } - - matched = clientIOWriteRegex.FindStringSubmatch(clientStr) - if len(matched) == 3 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } - - switch strings.ToLower(matched[2]) { - case "gb": - v = v * 1e9 - case "mb": - v = v * 1e6 - case "kb": - v = v * 1e3 - default: - return fmt.Errorf("can't parse units %q", matched[2]) - } - - c.ClientIOWrite.Set(float64(v)) - } - - var clientIOOps float64 - matched = clientIOOpsRegex.FindStringSubmatch(clientStr) - if len(matched) == 2 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } - - clientIOOps = float64(v) - } - - var clientIOReadOps, clientIOWriteOps float64 - matched = clientIOReadOpsRegex.FindStringSubmatch(clientStr) - if len(matched) == 2 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } - - clientIOReadOps = float64(v) - c.ClientIOReadOps.Set(clientIOReadOps) - } - - matched = clientIOWriteOpsRegex.FindStringSubmatch(clientStr) - if len(matched) == 2 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } - - clientIOWriteOps = float64(v) - c.ClientIOWriteOps.Set(clientIOWriteOps) - } - - // In versions older than Jewel, we directly get access to total - // client I/O. But in Jewel and newer the format is changed to - // separately display read and write IOPs. In such a case, we - // compute and set the total IOPs ourselves. - if clientIOOps == 0 { - clientIOOps = clientIOReadOps + clientIOWriteOps - } + clientIOWriteOps := float64(stats.PgMap.WriteOps) + c.ClientIOWriteOps.Set(clientIOWriteOps) + clientIOOps := clientIOReadOps + clientIOWriteOps c.ClientIOOps.Set(clientIOOps) return nil } -func (c *ClusterHealthCollector) collectRecoveryIO(recoveryStr string) error { - matched := recoveryIORateRegex.FindStringSubmatch(recoveryStr) - if len(matched) == 3 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } +func (c *ClusterHealthCollector) collectRecoveryIO(stats cephclient.CephStatus) error { + c.RecoveryIORate.Set(float64(stats.PgMap.RecoveryBps)) + c.RecoveryIOKeys.Set(float64(stats.PgMap.RecoveryKeysPerSec)) + c.RecoveryIOObjects.Set(float64(stats.PgMap.RecoveryObjectsPerSec)) - switch strings.ToLower(matched[2]) { - case "gb": - v = v * 1e9 - case "mb": - v = v * 1e6 - case "kb": - v = v * 1e3 - default: - return fmt.Errorf("can't parse units %q", matched[2]) - } - - c.RecoveryIORate.Set(float64(v)) - } - - matched = recoveryIOKeysRegex.FindStringSubmatch(recoveryStr) - if len(matched) == 2 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } - - c.RecoveryIOKeys.Set(float64(v)) - } - - matched = recoveryIOObjectsRegex.FindStringSubmatch(recoveryStr) - if len(matched) == 2 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } - - c.RecoveryIOObjects.Set(float64(v)) - } return nil } -func (c *ClusterHealthCollector) collectCacheIO(clientStr string) error { - matched := cacheFlushRateRegex.FindStringSubmatch(clientStr) - if len(matched) == 3 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } +func (c *ClusterHealthCollector) collectCacheIO(stats cephclient.CephStatus) error { + c.CacheFlushIORate.Set(float64(stats.PgMap.CacheFlushBps)) + c.CacheEvictIORate.Set(float64(stats.PgMap.CacheEvictBps)) + c.CachePromoteIOOps.Set(float64(stats.PgMap.CachePromoteBps)) - switch strings.ToLower(matched[2]) { - case "gb": - v = v * 1e9 - case "mb": - v = v * 1e6 - case "kb": - v = v * 1e3 - default: - return fmt.Errorf("can't parse units %q", matched[2]) - } - - c.CacheFlushIORate.Set(float64(v)) - } - - matched = cacheEvictRateRegex.FindStringSubmatch(clientStr) - if len(matched) == 3 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } - - switch strings.ToLower(matched[2]) { - case "gb": - v = v * 1e9 - case "mb": - v = v * 1e6 - case "kb": - v = v * 1e3 - default: - return fmt.Errorf("can't parse units %q", matched[2]) - } - - c.CacheEvictIORate.Set(float64(v)) - } - - matched = cachePromoteOpsRegex.FindStringSubmatch(clientStr) - if len(matched) == 2 { - v, err := strconv.Atoi(matched[1]) - if err != nil { - return err - } - - c.CachePromoteIOOps.Set(float64(v)) - } return nil } @@ -777,11 +584,17 @@ func (c *ClusterHealthCollector) Describe(ch chan<- *prometheus.Desc) { // Collect sends all the collected metrics to the provided prometheus channel. // It requires the caller to handle synchronization. func (c *ClusterHealthCollector) Collect(ch chan<- prometheus.Metric) { - if err := c.collect(); err != nil { + stats, err := cephclient.Status(c.context, c.clusterName) + if err != nil { + logger.Errorf("failed to get status: %+v", err) + return + } + + if err := c.collect(stats); err != nil { logger.Errorf("failed collecting cluster health metrics: %+v", err) } - if err := c.collectRecoveryClientIO(); err != nil { + if err := c.collectRecoveryClientIO(stats); err != nil { logger.Errorf("failed collecting cluster recovery/client io: %+v", err) } diff --git a/pkg/ceph/collectors/health_test.go b/pkg/ceph/collectors/health_test.go index 51823a26d..b857e0bb2 100644 --- a/pkg/ceph/collectors/health_test.go +++ b/pkg/ceph/collectors/health_test.go @@ -19,6 +19,7 @@ which has the same license. package collectors import ( + "fmt" "io/ioutil" "net/http" "net/http/httptest" @@ -29,6 +30,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/rook/rook/pkg/clusterd" exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" ) func TestClusterHealthCollector(t *testing.T) { @@ -38,203 +40,233 @@ func TestClusterHealthCollector(t *testing.T) { }{ { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "5 pgs degraded"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: 5 pgs degraded" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`degraded_pgs 5`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "15 pgs stuck degraded"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: 15 pgs stuck degraded" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`stuck_degraded_pgs 15`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "6 pgs unclean"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: 6 pgs unclean, 48 pgs stale" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`unclean_pgs 6`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "16 pgs stuck unclean"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: 16 pgs stuck unclean" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`stuck_unclean_pgs 16`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "7 pgs undersized"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: 7 pgs undersized" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`undersized_pgs 7`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "17 pgs stuck undersized"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: 17 pgs stuck undersized" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`stuck_undersized_pgs 17`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "8 pgs stale"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: 8 pgs stale" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`stale_pgs 8`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "18 pgs stuck stale"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: 18 pgs stuck stale" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`stuck_stale_pgs 18`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "recovery 10/20 objects degraded"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: recovery 10/20 objects degraded" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`degraded_objects 10`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "recovery 20/40 objects misplaced"}]} -}`, + { + "health": { + "checks": { + "PG_DEGRADED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "Degraded data redundancy: recovery 20/40 objects misplaced" + } + } + }, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`misplaced_objects 20`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 20, - "num_up_osds": 10, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - } -}`, + { + "osdmap": { + "osdmap": { + "num_osds": 20, + "num_up_osds": 10, + "num_in_osds": 0, + "num_remapped_pgs": 0 + } + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`osds_down 10`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 1200, - "num_up_osds": 1200, - "num_in_osds": 1190, - "num_remapped_pgs": 10 - } - }, - "health": {"summary": []} -}`, + { + "osdmap": { + "osdmap": { + "num_osds": 1200, + "num_up_osds": 1200, + "num_in_osds": 1190, + "num_remapped_pgs": 10 + } + }, + "health": {"summary": []} + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`osds 1200`), regexp.MustCompile(`osds_up 1200`), @@ -244,106 +276,86 @@ func TestClusterHealthCollector(t *testing.T) { }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 1200, - "num_up_osds": 1200, - "num_in_osds": 1190, - "num_remapped_pgs": 10 - } - }, - "health": { "overall_status": "HEALTH_OK" } }`, + { + "health": { + "checks": {}, + "status": "HEALTH_OK", + "overall_status": "HEALTH_OK" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`health_status 0`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 1200, - "num_up_osds": 1200, - "num_in_osds": 1190, - "num_remapped_pgs": 10 - } - }, - "health": { "overall_status": "HEALTH_WARN" } }`, + { + "health": { + "checks": {}, + "status": "HEALTH_WARN", + "overall_status": "HEALTH_WARN" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`health_status 1`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 1200, - "num_up_osds": 1200, - "num_in_osds": 1190, - "num_remapped_pgs": 10 - } - }, - "health": { "overall_status": "HEALTH_ERR" } }`, + { + "health": { + "checks": {}, + "status": "HEALTH_ERR", + "overall_status": "HEALTH_ERR" + } + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`health_status 2`), }, }, { input: ` -$ sudo ceph -s - cluster eff51be8-938a-4afa-b0d1-7a580b4ceb37 - health HEALTH_OK - monmap e3: 3 mons at {mon01,mon02,mon03} - recovery io 5779 MB/s, 4 keys/s, 1522 objects/s - client io 4273 kB/s rd, 2740 MB/s wr, 2863 op/s +{ + "pgmap": { + "read_bytes_sec": 1234, + "write_bytes_sec": 5678, + "read_op_per_sec": 10, + "write_op_per_sec": 20, + "recovering_bytes_per_sec": 123, + "recovering_objects_per_sec": 1522, + "recovering_keys_per_sec": 4, + "flush_bytes_sec": 2510, + "evict_bytes_sec": 6646, + "promote_op_per_sec": 55 + } +} `, regexes: []*regexp.Regexp{ - regexp.MustCompile(`recovery_io_bytes 5.779e`), + regexp.MustCompile(`recovery_io_bytes 123`), regexp.MustCompile(`recovery_io_keys 4`), regexp.MustCompile(`recovery_io_objects 1522`), - regexp.MustCompile(`client_io_ops 2863`), - regexp.MustCompile(`client_io_read_bytes 4.273e`), - regexp.MustCompile(`client_io_write_bytes 2.74e`), - }, - }, - { - input: ` -$ sudo ceph -s - cluster eff51be8-938a-4afa-b0d1-7a580b4ceb37 - health HEALTH_OK - monmap e3: 3 mons at {mon01,mon02,mon03} - recovery io 5779 MB/s, 4 keys/s, 1522 objects/s - client io 2863 op/s rd, 5847 op/s wr - cache io 251 MB/s flush, 6646 kB/s evict, 55 op/s promote -`, - regexes: []*regexp.Regexp{ - regexp.MustCompile(`recovery_io_bytes 5.779e`), - regexp.MustCompile(`recovery_io_keys 4`), - regexp.MustCompile(`recovery_io_objects 1522`), - regexp.MustCompile(`client_io_ops 8710`), - regexp.MustCompile(`client_io_read_ops 2863`), - regexp.MustCompile(`client_io_write_ops 5847`), - regexp.MustCompile(`cache_flush_io_bytes 2.51e`), - regexp.MustCompile(`cache_evict_io_bytes 6.646e`), + regexp.MustCompile(`client_io_ops 30`), + regexp.MustCompile(`client_io_read_bytes 1234`), + regexp.MustCompile(`client_io_write_bytes 5678`), + regexp.MustCompile(`cache_flush_io_bytes 2510`), + regexp.MustCompile(`cache_evict_io_bytes 6646`), regexp.MustCompile(`cache_promote_io_ops 55`), }, }, { input: ` -{ - "osdmap": { - "osdmap": { - "num_osds": 0, - "num_up_osds": 0, - "num_in_osds": 0, - "num_remapped_pgs": 0 - } - }, - "pgmap": { "num_pgs": 52000 }, - "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "7 pgs undersized"}]} -}`, + { + "osdmap": { + "osdmap": { + "num_osds": 0, + "num_up_osds": 0, + "num_in_osds": 0, + "num_remapped_pgs": 0 + } + }, + "pgmap": { "num_pgs": 52000 }, + "health": {"summary": [{"severity": "HEALTH_WARN", "summary": "7 pgs undersized"}]} + }`, regexes: []*regexp.Regexp{ regexp.MustCompile(`total_pgs 52000`), }, @@ -380,8 +392,7 @@ $ sudo ceph -s for _, re := range tt.regexes { if !re.Match(buf) { - // FIX: Add back the metrics - // assert.True(t, re.Match(buf), fmt.Sprintf("failed matching: %q", re)) + assert.True(t, re.Match(buf), fmt.Sprintf("failed matching: %q", re)) } } }()