feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

monitoring: update metrics collection for new Ceph Luminous formats

Browse Source
This commit is contained in:
Jared Watts
2017-08-11 15:50:55 -07:00
parent 815227991f
commit ed7f757c06
6 changed files with 295 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/status.go
+1 -1
View File
@@ -39,7 +39,7 @@ func (h *Handler) GetStatusDetails(w http.ResponseWriter, r *http.Request) {
checks[i] = model.StatusSummary{
Name: name,
Status: ceph.HealthToModelHealthStatus(check.Severity),
Message: check.Message,
Message: check.Summary.Message,
}
i++
}
pkg/api/status_test.go
+1 -1
View File
@@ -30,7 +30,7 @@ import (
const (
// this JSON was generated from the command: ceph status
cephStatusResponseRaw = `{"fsid":"d64ecb40-3e93-44e8-9957-2faba55c56de","health":{"checks":{"TEST_MSG":{"severity":"HEALTH_WARN","message":"too many PGs per OSD (2048 > max 300)"}},"status":"HEALTH_WARN"},"election_epoch":6,"quorum":[0],"quorum_names":["rook-ceph-mon0"],"monmap":{"epoch":3,"fsid":"d64ecb40-3e93-44e8-9957-2faba55c56de","modified":"2017-08-01 16:50:42.901253","created":"2017-08-01 16:50:30.751733","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"127.0.0.1:6790/0","public_addr":"127.0.0.1:6790/0"}]},"osdmap":{"osdmap":{"epoch":8,"num_osds":10,"num_up_osds":9,"num_in_osds":9,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"created+peering","count": 100},{"state_name": "active+clean","count": 2048}],"num_pgs":2148,"num_pools":0,"num_objects":0,"data_bytes":0,"bytes_used":123,"bytes_avail":234,"bytes_total":345},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":4109,"active_name":"rook-ceph-mgr0","active_addr":"172.17.0.8:6800/13","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}`
cephStatusResponseRaw = `{"fsid":"d64ecb40-3e93-44e8-9957-2faba55c56de","health":{"checks":{"TEST_MSG":{"severity":"HEALTH_WARN","summary":{"message":"too many PGs per OSD (2048 > max 300)"}}},"status":"HEALTH_WARN"},"election_epoch":6,"quorum":[0],"quorum_names":["rook-ceph-mon0"],"monmap":{"epoch":3,"fsid":"d64ecb40-3e93-44e8-9957-2faba55c56de","modified":"2017-08-01 16:50:42.901253","created":"2017-08-01 16:50:30.751733","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"127.0.0.1:6790/0","public_addr":"127.0.0.1:6790/0"}]},"osdmap":{"osdmap":{"epoch":8,"num_osds":10,"num_up_osds":9,"num_in_osds":9,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"created+peering","count": 100},{"state_name": "active+clean","count": 2048}],"num_pgs":2148,"num_pools":0,"num_objects":0,"data_bytes":0,"bytes_used":123,"bytes_avail":234,"bytes_total":345},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":4109,"active_name":"rook-ceph-mgr0","active_addr":"172.17.0.8:6800/13","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}`
// this JSON was generated from the command: ceph time-sync-status
cephTimeStatusReponseRaw = `{"time_skew_status":{"rook-ceph-mon0":{"skew":0.000000,"latency":0.000000,"health":"HEALTH_OK"}},"timechecks":{"epoch":6,"round":6,"round_status":"finished"}}`
)
pkg/ceph/client/status.go
+20 -8
View File
@@ -59,7 +59,9 @@ type HealthStatus struct {
type CheckMessage struct {
Severity string `json:"severity"`
Message string `json:"message"`
Summary struct {
Message string `json:"message"`
} `json:"summary"`
}
type MonMap struct {
@@ -95,13 +97,23 @@ type OsdMap struct {
}
type PgMap struct {
PgsByState []PgStateEntry `json:"pgs_by_state"`
Version int `json:"version"`
NumPgs int `json:"num_pgs"`
DataBytes uint64 `json:"data_bytes"`
UsedBytes uint64 `json:"bytes_used"`
AvailableBytes uint64 `json:"bytes_avail"`
TotalBytes uint64 `json:"bytes_total"`
PgsByState []PgStateEntry `json:"pgs_by_state"`
Version int `json:"version"`
NumPgs int `json:"num_pgs"`
DataBytes uint64 `json:"data_bytes"`
UsedBytes uint64 `json:"bytes_used"`
AvailableBytes uint64 `json:"bytes_avail"`
TotalBytes uint64 `json:"bytes_total"`
ReadBps uint64 `json:"read_bytes_sec"`
WriteBps uint64 `json:"write_bytes_sec"`
ReadOps uint64 `json:"read_op_per_sec"`
WriteOps uint64 `json:"write_op_per_sec"`
RecoveryBps uint64 `json:"recovering_bytes_per_sec"`
RecoveryObjectsPerSec uint64 `json:"recovering_objects_per_sec"`
RecoveryKeysPerSec uint64 `json:"recovering_keys_per_sec"`
CacheFlushBps uint64 `json:"flush_bytes_sec"`
CacheEvictBps uint64 `json:"evict_bytes_sec"`
CachePromoteBps uint64 `json:"promote_op_per_sec"`
}
type PgStateEntry struct {
pkg/ceph/client/status_test.go
+12 -12
View File
@@ -23,8 +23,8 @@ import (
)
const (
// this JSON was generated from the mon_command "status", ExecuteMonCommand(conn, map[string]interface{}{"prefix": "status"})
CephStatusResponseRaw = `{"fsid":"3ca36e66-bd6a-49d4-a976-cdac216b78c3","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","message":"1 osds down"},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","message":"1 host (1 osds) down"},"OSD_ROOT_DOWN":{"severity":"HEALTH_WARN","message":"1 root (1 osds) down"},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","message":"Reduced data availability: 100 pgs stale"}},"status":"HEALTH_WARN"},"election_epoch":22,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon1","rook-ceph-mon2"],"monmap":{"epoch":1,"fsid":"3ca36e66-bd6a-49d4-a976-cdac216b78c3","modified":"2017-07-28 02:39:57.490676","created":"2017-07-28 02:39:57.490676","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.0.0.13:6790/0","public_addr":"10.0.0.13:6790/0"},{"rank":1,"name":"rook-ceph-mon1","addr":"10.0.0.102:6790/0","public_addr":"10.0.0.102:6790/0"},{"rank":2,"name":"rook-ceph-mon2","addr":"10.0.0.227:6790/0","public_addr":"10.0.0.227:6790/0"}]},"osdmap":{"osdmap":{"epoch":13,"num_osds":1,"num_up_osds":0,"num_in_osds":1,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":100}],"num_pgs":100,"num_pools":1,"num_objects":0,"data_bytes":0,"bytes_used":3464249344,"bytes_avail":13829283840,"bytes_total":17293533184},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":4112,"active_name":"rook-ceph-mgr0","active_addr":"172.17.0.10:6800/13","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}`
// this JSON was generated from `ceph status -f json`, using Ceph Luminous 12.1.3
CephStatusResponseRaw = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6790/0","public_addr":"10.3.0.45:6790/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6790/0","public_addr":"10.3.0.249:6790/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6790/0","public_addr":"10.3.0.252:6790/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}`
)
func TestStatusMarshal(t *testing.T) {
@@ -36,24 +36,24 @@ func TestStatusMarshal(t *testing.T) {
assert.Equal(t, "HEALTH_WARN", status.Health.Status)
assert.Equal(t, 4, len(status.Health.Checks))
assert.Equal(t, "HEALTH_WARN", status.Health.Checks["OSD_DOWN"].Severity)
assert.Equal(t, "1 osds down", status.Health.Checks["OSD_DOWN"].Message)
assert.Equal(t, "HEALTH_WARN", status.Health.Checks["OSD_ROOT_DOWN"].Severity)
assert.Equal(t, "1 root (1 osds) down", status.Health.Checks["OSD_ROOT_DOWN"].Message)
assert.Equal(t, "1 osds down", status.Health.Checks["OSD_DOWN"].Summary.Message)
assert.Equal(t, "HEALTH_WARN", status.Health.Checks["OSD_HOST_DOWN"].Severity)
assert.Equal(t, "1 host (1 osds) down", status.Health.Checks["OSD_HOST_DOWN"].Summary.Message)
// verify some Mon map fields
assert.Equal(t, 1, status.MonMap.Epoch)
assert.Equal(t, 3, status.MonMap.Epoch)
assert.Equal(t, "rook-ceph-mon0", status.MonMap.Mons[0].Name)
assert.Equal(t, "10.0.0.13:6790/0", status.MonMap.Mons[0].Address)
assert.Equal(t, "10.3.0.45:6790/0", status.MonMap.Mons[0].Address)
// verify some OSD map fields
assert.Equal(t, 1, status.OsdMap.OsdMap.NumOsd)
assert.Equal(t, 0, status.OsdMap.OsdMap.NumUpOsd)
assert.Equal(t, 2, status.OsdMap.OsdMap.NumOsd)
assert.Equal(t, 1, status.OsdMap.OsdMap.NumUpOsd)
assert.False(t, status.OsdMap.OsdMap.Full)
assert.True(t, status.OsdMap.OsdMap.NearFull)
// verify some PG map fields
assert.Equal(t, 100, status.PgMap.NumPgs)
assert.Equal(t, uint64(3464249344), status.PgMap.UsedBytes)
assert.Equal(t, 100, status.PgMap.PgsByState[0].Count)
assert.Equal(t, 200, status.PgMap.NumPgs)
assert.Equal(t, uint64(13611479040), status.PgMap.UsedBytes)
assert.Equal(t, 101, status.PgMap.PgsByState[0].Count)
assert.Equal(t, "stale+active+clean", status.PgMap.PgsByState[0].StateName)
}
pkg/ceph/collectors/health.go
+44 -231
View File
@@ -20,12 +20,8 @@ which has the same license.
package collectors
import (
"bufio"
"bytes"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/prometheus/client_golang/prometheus"
@@ -33,22 +29,6 @@ import (
"github.com/rook/rook/pkg/clusterd"
)
var (
recoveryIORateRegex = regexp.MustCompile(`(\d+) (\w{2})/s`)
recoveryIOKeysRegex = regexp.MustCompile(`(\d+) keys/s`)
recoveryIOObjectsRegex = regexp.MustCompile(`(\d+) objects/s`)
clientIOReadRegex = regexp.MustCompile(`(\d+) ([kKmMgG][bB])/s rd`)
clientIOWriteRegex = regexp.MustCompile(`(\d+) ([kKmMgG][bB])/s wr`)
clientIOReadOpsRegex = regexp.MustCompile(`(\d+) op/s rd`)
clientIOWriteOpsRegex = regexp.MustCompile(`(\d+) op/s wr`)
cacheFlushRateRegex = regexp.MustCompile(`(\d+) ([kKmMgG][bB])/s flush`)
cacheEvictRateRegex = regexp.MustCompile(`(\d+) ([kKmMgG][bB])/s evict`)
cachePromoteOpsRegex = regexp.MustCompile(`(\d+) op/s promote`)
// Older versions of Ceph, hammer (v0.94) and below, support this format.
clientIOOpsRegex = regexp.MustCompile(`(\d+) op/s[^ \w]*$`)
)
// ClusterHealthCollector collects information about the health of an overall cluster.
// It surfaces changes in the ceph parameters unlike data usage that ClusterUsageCollector
// does.
@@ -399,11 +379,7 @@ func (c *ClusterHealthCollector) metricsList() []prometheus.Metric {
}
}
func (c *ClusterHealthCollector) collect() error {
stats, err := cephclient.Status(c.context, c.clusterName)
if err != nil {
return err
}
func (c *ClusterHealthCollector) collect(stats cephclient.CephStatus) error {
for _, metric := range c.metricsList() {
if gauge, ok := metric.(prometheus.Gauge); ok {
@@ -436,7 +412,7 @@ func (c *ClusterHealthCollector) collect() error {
)
for _, m := range stats.Health.Checks {
matched := degradedRegex.FindStringSubmatch(m.Message)
matched := degradedRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -445,7 +421,7 @@ func (c *ClusterHealthCollector) collect() error {
c.DegradedPGs.Set(float64(v))
}
matched = stuckDegradedRegex.FindStringSubmatch(m.Message)
matched = stuckDegradedRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -454,7 +430,7 @@ func (c *ClusterHealthCollector) collect() error {
c.StuckDegradedPGs.Set(float64(v))
}
matched = uncleanRegex.FindStringSubmatch(m.Message)
matched = uncleanRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -463,7 +439,7 @@ func (c *ClusterHealthCollector) collect() error {
c.UncleanPGs.Set(float64(v))
}
matched = stuckUncleanRegex.FindStringSubmatch(m.Message)
matched = stuckUncleanRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -472,7 +448,7 @@ func (c *ClusterHealthCollector) collect() error {
c.StuckUncleanPGs.Set(float64(v))
}
matched = undersizedRegex.FindStringSubmatch(m.Message)
matched = undersizedRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -481,7 +457,7 @@ func (c *ClusterHealthCollector) collect() error {
c.UndersizedPGs.Set(float64(v))
}
matched = stuckUndersizedRegex.FindStringSubmatch(m.Message)
matched = stuckUndersizedRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -490,7 +466,7 @@ func (c *ClusterHealthCollector) collect() error {
c.StuckUndersizedPGs.Set(float64(v))
}
matched = staleRegex.FindStringSubmatch(m.Message)
matched = staleRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -499,7 +475,7 @@ func (c *ClusterHealthCollector) collect() error {
c.StalePGs.Set(float64(v))
}
matched = stuckStaleRegex.FindStringSubmatch(m.Message)
matched = stuckStaleRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -508,7 +484,7 @@ func (c *ClusterHealthCollector) collect() error {
c.StuckStalePGs.Set(float64(v))
}
matched = degradedObjectsRegex.FindStringSubmatch(m.Message)
matched = degradedObjectsRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 3 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -517,7 +493,7 @@ func (c *ClusterHealthCollector) collect() error {
c.DegradedObjectsCount.Set(float64(v))
}
matched = misplacedObjectsRegex.FindStringSubmatch(m.Message)
matched = misplacedObjectsRegex.FindStringSubmatch(m.Summary.Message)
if len(matched) == 3 {
v, err := strconv.Atoi(matched[1])
if err != nil {
@@ -549,220 +525,51 @@ func (c *ClusterHealthCollector) collect() error {
return nil
}
func (c *ClusterHealthCollector) collectRecoveryClientIO() error {
buf, err := cephclient.StatusPlain(c.context, c.clusterName)
if err != nil {
func (c *ClusterHealthCollector) collectRecoveryClientIO(stats cephclient.CephStatus) error {
if err := c.collectRecoveryIO(stats); err != nil {
return err
}
sc := bufio.NewScanner(bytes.NewReader(buf))
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
switch {
case strings.HasPrefix(line, "recovery io"):
if err := c.collectRecoveryIO(line); err != nil {
return err
}
case strings.HasPrefix(line, "client io"):
if err := c.collectClientIO(line); err != nil {
return err
}
case strings.HasPrefix(line, "cache io"):
if err := c.collectCacheIO(line); err != nil {
return err
}
}
if err := c.collectClientIO(stats); err != nil {
return err
}
if err := c.collectCacheIO(stats); err != nil {
return err
}
return nil
}
func (c *ClusterHealthCollector) collectClientIO(clientStr string) error {
matched := clientIOReadRegex.FindStringSubmatch(clientStr)
if len(matched) == 3 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
func (c *ClusterHealthCollector) collectClientIO(stats cephclient.CephStatus) error {
c.ClientIORead.Set(float64(stats.PgMap.ReadBps))
c.ClientIOWrite.Set(float64(stats.PgMap.WriteBps))
switch strings.ToLower(matched[2]) {
case "gb":
v = v * 1e9
case "mb":
v = v * 1e6
case "kb":
v = v * 1e3
default:
return fmt.Errorf("can't parse units %q", matched[2])
}
clientIOReadOps := float64(stats.PgMap.ReadOps)
c.ClientIOReadOps.Set(clientIOReadOps)
c.ClientIORead.Set(float64(v))
}
matched = clientIOWriteRegex.FindStringSubmatch(clientStr)
if len(matched) == 3 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
switch strings.ToLower(matched[2]) {
case "gb":
v = v * 1e9
case "mb":
v = v * 1e6
case "kb":
v = v * 1e3
default:
return fmt.Errorf("can't parse units %q", matched[2])
}
c.ClientIOWrite.Set(float64(v))
}
var clientIOOps float64
matched = clientIOOpsRegex.FindStringSubmatch(clientStr)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
clientIOOps = float64(v)
}
var clientIOReadOps, clientIOWriteOps float64
matched = clientIOReadOpsRegex.FindStringSubmatch(clientStr)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
clientIOReadOps = float64(v)
c.ClientIOReadOps.Set(clientIOReadOps)
}
matched = clientIOWriteOpsRegex.FindStringSubmatch(clientStr)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
clientIOWriteOps = float64(v)
c.ClientIOWriteOps.Set(clientIOWriteOps)
}
// In versions older than Jewel, we directly get access to total
// client I/O. But in Jewel and newer the format is changed to
// separately display read and write IOPs. In such a case, we
// compute and set the total IOPs ourselves.
if clientIOOps == 0 {
clientIOOps = clientIOReadOps + clientIOWriteOps
}
clientIOWriteOps := float64(stats.PgMap.WriteOps)
c.ClientIOWriteOps.Set(clientIOWriteOps)
clientIOOps := clientIOReadOps + clientIOWriteOps
c.ClientIOOps.Set(clientIOOps)
return nil
}
func (c *ClusterHealthCollector) collectRecoveryIO(recoveryStr string) error {
matched := recoveryIORateRegex.FindStringSubmatch(recoveryStr)
if len(matched) == 3 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
func (c *ClusterHealthCollector) collectRecoveryIO(stats cephclient.CephStatus) error {
c.RecoveryIORate.Set(float64(stats.PgMap.RecoveryBps))
c.RecoveryIOKeys.Set(float64(stats.PgMap.RecoveryKeysPerSec))
c.RecoveryIOObjects.Set(float64(stats.PgMap.RecoveryObjectsPerSec))
switch strings.ToLower(matched[2]) {
case "gb":
v = v * 1e9
case "mb":
v = v * 1e6
case "kb":
v = v * 1e3
default:
return fmt.Errorf("can't parse units %q", matched[2])
}
c.RecoveryIORate.Set(float64(v))
}
matched = recoveryIOKeysRegex.FindStringSubmatch(recoveryStr)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
c.RecoveryIOKeys.Set(float64(v))
}
matched = recoveryIOObjectsRegex.FindStringSubmatch(recoveryStr)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
c.RecoveryIOObjects.Set(float64(v))
}
return nil
}
func (c *ClusterHealthCollector) collectCacheIO(clientStr string) error {
matched := cacheFlushRateRegex.FindStringSubmatch(clientStr)
if len(matched) == 3 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
func (c *ClusterHealthCollector) collectCacheIO(stats cephclient.CephStatus) error {
c.CacheFlushIORate.Set(float64(stats.PgMap.CacheFlushBps))
c.CacheEvictIORate.Set(float64(stats.PgMap.CacheEvictBps))
c.CachePromoteIOOps.Set(float64(stats.PgMap.CachePromoteBps))
switch strings.ToLower(matched[2]) {
case "gb":
v = v * 1e9
case "mb":
v = v * 1e6
case "kb":
v = v * 1e3
default:
return fmt.Errorf("can't parse units %q", matched[2])
}
c.CacheFlushIORate.Set(float64(v))
}
matched = cacheEvictRateRegex.FindStringSubmatch(clientStr)
if len(matched) == 3 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
switch strings.ToLower(matched[2]) {
case "gb":
v = v * 1e9
case "mb":
v = v * 1e6
case "kb":
v = v * 1e3
default:
return fmt.Errorf("can't parse units %q", matched[2])
}
c.CacheEvictIORate.Set(float64(v))
}
matched = cachePromoteOpsRegex.FindStringSubmatch(clientStr)
if len(matched) == 2 {
v, err := strconv.Atoi(matched[1])
if err != nil {
return err
}
c.CachePromoteIOOps.Set(float64(v))
}
return nil
}
@@ -777,11 +584,17 @@ func (c *ClusterHealthCollector) Describe(ch chan<- *prometheus.Desc) {
// Collect sends all the collected metrics to the provided prometheus channel.
// It requires the caller to handle synchronization.
func (c *ClusterHealthCollector) Collect(ch chan<- prometheus.Metric) {
if err := c.collect(); err != nil {
stats, err := cephclient.Status(c.context, c.clusterName)
if err != nil {
logger.Errorf("failed to get status: %+v", err)
return
}
if err := c.collect(stats); err != nil {
logger.Errorf("failed collecting cluster health metrics: %+v", err)
}
if err := c.collectRecoveryClientIO(); err != nil {
if err := c.collectRecoveryClientIO(stats); err != nil {
logger.Errorf("failed collecting cluster recovery/client io: %+v", err)
}
pkg/ceph/collectors/health_test.go
+217 -206
View File
@@ -19,6 +19,7 @@ which has the same license.
package collectors
import (
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
@@ -29,6 +30,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/rook/rook/pkg/clusterd"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
)
func TestClusterHealthCollector(t *testing.T) {
@@ -38,203 +40,233 @@ func TestClusterHealthCollector(t *testing.T) {
}{
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "5 pgs degraded"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: 5 pgs degraded"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`degraded_pgs 5`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "15 pgs stuck degraded"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: 15 pgs stuck degraded"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`stuck_degraded_pgs 15`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "6 pgs unclean"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: 6 pgs unclean, 48 pgs stale"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`unclean_pgs 6`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "16 pgs stuck unclean"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: 16 pgs stuck unclean"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`stuck_unclean_pgs 16`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "7 pgs undersized"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: 7 pgs undersized"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`undersized_pgs 7`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "17 pgs stuck undersized"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: 17 pgs stuck undersized"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`stuck_undersized_pgs 17`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "8 pgs stale"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: 8 pgs stale"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`stale_pgs 8`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "18 pgs stuck stale"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: 18 pgs stuck stale"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`stuck_stale_pgs 18`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "recovery 10/20 objects degraded"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: recovery 10/20 objects degraded"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`degraded_objects 10`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "recovery 20/40 objects misplaced"}]}
}`,
{
"health": {
"checks": {
"PG_DEGRADED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Degraded data redundancy: recovery 20/40 objects misplaced"
}
}
},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`misplaced_objects 20`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 20,
"num_up_osds": 10,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
}
}`,
{
"osdmap": {
"osdmap": {
"num_osds": 20,
"num_up_osds": 10,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`osds_down 10`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 1200,
"num_up_osds": 1200,
"num_in_osds": 1190,
"num_remapped_pgs": 10
}
},
"health": {"summary": []}
}`,
{
"osdmap": {
"osdmap": {
"num_osds": 1200,
"num_up_osds": 1200,
"num_in_osds": 1190,
"num_remapped_pgs": 10
}
},
"health": {"summary": []}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`osds 1200`),
regexp.MustCompile(`osds_up 1200`),
@@ -244,106 +276,86 @@ func TestClusterHealthCollector(t *testing.T) {
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 1200,
"num_up_osds": 1200,
"num_in_osds": 1190,
"num_remapped_pgs": 10
}
},
"health": { "overall_status": "HEALTH_OK" } }`,
{
"health": {
"checks": {},
"status": "HEALTH_OK",
"overall_status": "HEALTH_OK"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`health_status 0`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 1200,
"num_up_osds": 1200,
"num_in_osds": 1190,
"num_remapped_pgs": 10
}
},
"health": { "overall_status": "HEALTH_WARN" } }`,
{
"health": {
"checks": {},
"status": "HEALTH_WARN",
"overall_status": "HEALTH_WARN"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`health_status 1`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 1200,
"num_up_osds": 1200,
"num_in_osds": 1190,
"num_remapped_pgs": 10
}
},
"health": { "overall_status": "HEALTH_ERR" } }`,
{
"health": {
"checks": {},
"status": "HEALTH_ERR",
"overall_status": "HEALTH_ERR"
}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`health_status 2`),
},
},
{
input: `
$ sudo ceph -s
cluster eff51be8-938a-4afa-b0d1-7a580b4ceb37
health HEALTH_OK
monmap e3: 3 mons at {mon01,mon02,mon03}
recovery io 5779 MB/s, 4 keys/s, 1522 objects/s
client io 4273 kB/s rd, 2740 MB/s wr, 2863 op/s
{
"pgmap": {
"read_bytes_sec": 1234,
"write_bytes_sec": 5678,
"read_op_per_sec": 10,
"write_op_per_sec": 20,
"recovering_bytes_per_sec": 123,
"recovering_objects_per_sec": 1522,
"recovering_keys_per_sec": 4,
"flush_bytes_sec": 2510,
"evict_bytes_sec": 6646,
"promote_op_per_sec": 55
}
}
`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`recovery_io_bytes 5.779e`),
regexp.MustCompile(`recovery_io_bytes 123`),
regexp.MustCompile(`recovery_io_keys 4`),
regexp.MustCompile(`recovery_io_objects 1522`),
regexp.MustCompile(`client_io_ops 2863`),
regexp.MustCompile(`client_io_read_bytes 4.273e`),
regexp.MustCompile(`client_io_write_bytes 2.74e`),
},
},
{
input: `
$ sudo ceph -s
cluster eff51be8-938a-4afa-b0d1-7a580b4ceb37
health HEALTH_OK
monmap e3: 3 mons at {mon01,mon02,mon03}
recovery io 5779 MB/s, 4 keys/s, 1522 objects/s
client io 2863 op/s rd, 5847 op/s wr
cache io 251 MB/s flush, 6646 kB/s evict, 55 op/s promote
`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`recovery_io_bytes 5.779e`),
regexp.MustCompile(`recovery_io_keys 4`),
regexp.MustCompile(`recovery_io_objects 1522`),
regexp.MustCompile(`client_io_ops 8710`),
regexp.MustCompile(`client_io_read_ops 2863`),
regexp.MustCompile(`client_io_write_ops 5847`),
regexp.MustCompile(`cache_flush_io_bytes 2.51e`),
regexp.MustCompile(`cache_evict_io_bytes 6.646e`),
regexp.MustCompile(`client_io_ops 30`),
regexp.MustCompile(`client_io_read_bytes 1234`),
regexp.MustCompile(`client_io_write_bytes 5678`),
regexp.MustCompile(`cache_flush_io_bytes 2510`),
regexp.MustCompile(`cache_evict_io_bytes 6646`),
regexp.MustCompile(`cache_promote_io_ops 55`),
},
},
{
input: `
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"pgmap": { "num_pgs": 52000 },
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "7 pgs undersized"}]}
}`,
{
"osdmap": {
"osdmap": {
"num_osds": 0,
"num_up_osds": 0,
"num_in_osds": 0,
"num_remapped_pgs": 0
}
},
"pgmap": { "num_pgs": 52000 },
"health": {"summary": [{"severity": "HEALTH_WARN", "summary": "7 pgs undersized"}]}
}`,
regexes: []*regexp.Regexp{
regexp.MustCompile(`total_pgs 52000`),
},
@@ -380,8 +392,7 @@ $ sudo ceph -s
for _, re := range tt.regexes {
if !re.Match(buf) {
// FIX: Add back the metrics
// assert.True(t, re.Match(buf), fmt.Sprintf("failed matching: %q", re))
assert.True(t, re.Match(buf), fmt.Sprintf("failed matching: %q", re))
}
}
}()