feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #10578 from microyahoo/optimize

Browse Source 
osd: optimize device probe
This commit is contained in:
Blaine Gardner authored and GitHub committed 2022-09-20 10:34:29 -06:00
2 parents 1036748506 9a78f6af07
commit dbb98224bc
6 files changed
+34 -28
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/daemon/ceph/client/status.go
+5 -6
View File
@@ -48,12 +48,10 @@ type CephStatus struct {
Quorum []int `json:"quorum"`
QuorumNames []string `json:"quorum_names"`
MonMap MonMap `json:"monmap"`
OsdMap struct {
OsdMap OsdMap `json:"osdmap"`
} `json:"osdmap"`
PgMap PgMap `json:"pgmap"`
MgrMap MgrMap `json:"mgrmap"`
Fsmap Fsmap `json:"fsmap"`
OsdMap OsdMap `json:"osdmap"`
PgMap PgMap `json:"pgmap"`
MgrMap MgrMap `json:"mgrmap"`
Fsmap Fsmap `json:"fsmap"`
}
type HealthStatus struct {
@@ -72,6 +70,7 @@ type Summary struct {
type MonMap struct {
Epoch int `json:"epoch"`
NumMons int `josn:"num_mons"`
FSID string `json:"fsid"`
CreatedTime string `json:"created"`
ModifiedTime string `json:"modified"`
pkg/daemon/ceph/client/status_test.go
+5 -5
View File
@@ -24,7 +24,7 @@ import (
const (
// this JSON was generated from `ceph status -f json`, using Ceph Luminous 12.1.3
CephStatusResponseRaw = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}`
CephStatusResponseRaw = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}`
)
var (
@@ -70,10 +70,10 @@ func TestStatusMarshal(t *testing.T) {
assert.Equal(t, "10.3.0.45:6789/0", status.MonMap.Mons[0].Address)
// verify some OSD map fields
assert.Equal(t, 2, status.OsdMap.OsdMap.NumOsd)
assert.Equal(t, 1, status.OsdMap.OsdMap.NumUpOsd)
assert.False(t, status.OsdMap.OsdMap.Full)
assert.True(t, status.OsdMap.OsdMap.NearFull)
assert.Equal(t, 2, status.OsdMap.NumOsd)
assert.Equal(t, 1, status.OsdMap.NumUpOsd)
assert.False(t, status.OsdMap.Full)
assert.True(t, status.OsdMap.NearFull)
// verify some PG map fields
assert.Equal(t, 200, status.PgMap.NumPgs)
pkg/daemon/discover/discover.go
+14 -10
View File
@@ -93,7 +93,7 @@ func Run(ctx context.Context, context *clusterd.Context, probeInterval time.Dura
return err
}
udevEvents := make(chan string)
udevEvents := make(chan struct{})
go udevBlockMonitor(udevEvents, udevEventPeriod)
for {
select {
@@ -148,11 +148,11 @@ func matchUdevEvent(text string, matches, exclusions []string) (bool, error) {
// Scans `udevadm monitor` output for block sub-system events. Each line of
// output matching a set of substrings is sent to the provided channel. An event
// is returned if it passes any matches tests, and passes all exclusion tests.
func rawUdevBlockMonitor(c chan string, matches, exclusions []string) {
func rawUdevBlockMonitor(c chan struct{}, matches, exclusions []string) {
defer close(c)
// stdbuf -oL performs line buffered output
cmd := exec.Command("stdbuf", "-oL", "udevadm", "monitor", "-u", "-k", "-s", "block")
cmd := exec.Command("stdbuf", "-oL", "udevadm", "monitor", "-u", "-s", "block")
stdout, err := cmd.StdoutPipe()
if err != nil {
logger.Warningf("Cannot open udevadm stdout: %v", err)
@@ -176,7 +176,7 @@ func rawUdevBlockMonitor(c chan string, matches, exclusions []string) {
return
}
if match {
c <- text
c <- struct{}{}
}
}
@@ -189,13 +189,13 @@ func rawUdevBlockMonitor(c chan string, matches, exclusions []string) {
// Monitors udev for block device changes, and collapses these events such that
// only one event is emitted per period in order to deal with flapping.
func udevBlockMonitor(c chan string, period time.Duration) {
func udevBlockMonitor(c chan struct{}, period time.Duration) {
defer close(c)
var udevFilter []string
// return any add or remove events, but none that match device mapper
// events. string matching is case-insensitive
events := make(chan string)
events := make(chan struct{})
// get discoverDaemonUdevBlacklist from the environment variable
// if user doesn't provide any regex; generate the default regex
@@ -211,16 +211,20 @@ func udevBlockMonitor(c chan string, period time.Duration) {
[]string{"(?i)add", "(?i)remove"},
udevFilter)
timeout := time.NewTimer(period)
defer timeout.Stop()
for {
event, ok := <-events
_, ok := <-events
if !ok {
return
}
timeout := time.NewTimer(period)
if !timeout.Stop() {
<-timeout.C
}
timeout.Reset(period)
for {
select {
case <-timeout.C:
break
case _, ok := <-events:
if !ok {
return
@@ -229,7 +233,7 @@ func udevBlockMonitor(c chan string, period time.Duration) {
}
break
}
c <- event
c <- struct{}{}
}
}
tests/framework/clients/cluster.go
+2 -2
View File
@@ -42,11 +42,11 @@ func IsClusterHealthy(testClient *TestClient, namespace string) (bool, error) {
}
// verify there are OSDs and they are all up/in
totalOSDs := status.OsdMap.OsdMap.NumOsd
totalOSDs := status.OsdMap.NumOsd
if totalOSDs == 0 {
return false, fmt.Errorf("no OSDs: %+v", status)
}
if status.OsdMap.OsdMap.NumInOsd != totalOSDs || status.OsdMap.OsdMap.NumUpOsd != totalOSDs {
if status.OsdMap.NumInOsd != totalOSDs || status.OsdMap.NumUpOsd != totalOSDs {
return false, fmt.Errorf("not all OSDs are up/in: %+v", status)
}
tests/framework/utils/k8s_helper.go
+3 -3
View File
@@ -866,8 +866,8 @@ func (k8sh *K8sHelper) IsPodInError(podNamePattern, namespace, reason, containin
return false
}
// GetPodHostID returns HostIP address of a pod
func (k8sh *K8sHelper) GetPodHostID(podNamePattern string, namespace string) (string, error) {
// GetPodHostIP returns HostIP address of a pod
func (k8sh *K8sHelper) GetPodHostIP(podNamePattern string, namespace string) (string, error) {
ctx := context.TODO()
listOpts := metav1.ListOptions{LabelSelector: "app=" + podNamePattern}
podList, err := k8sh.Clientset.CoreV1().Pods(namespace).List(ctx, listOpts)
@@ -1275,7 +1275,7 @@ func (k8sh *K8sHelper) getInternalRGWServiceURL(storeName string, namespace stri
// GetRGWServiceURL returns URL of ceph RGW service in the cluster
func (k8sh *K8sHelper) getExternalRGWServiceURL(storeName string, namespace string) (string, error) {
hostip, err := k8sh.GetPodHostID("rook-ceph-rgw", namespace)
hostip, err := k8sh.GetPodHostIP("rook-ceph-rgw", namespace)
if err != nil {
return "", fmt.Errorf("RGW pods not found. %+v", err)
}
tests/integration/ceph_base_deploy_test.go
+5 -2
View File
@@ -56,11 +56,14 @@ func checkIfRookClusterIsInstalled(s *suite.Suite, k8sh *utils.K8sHelper, opName
func checkIfRookClusterIsHealthy(s *suite.Suite, testClient *clients.TestClient, clusterNamespace string) {
logger.Infof("Testing cluster %s health", clusterNamespace)
var err error
var (
err error
healthy bool
)
retryCount := 0
for retryCount < utils.RetryLoop {
healthy, err := clients.IsClusterHealthy(testClient, clusterNamespace)
healthy, err = clients.IsClusterHealthy(testClient, clusterNamespace)
if healthy {
logger.Infof("cluster %s is healthy", clusterNamespace)
return