diff --git a/pkg/daemon/ceph/client/status.go b/pkg/daemon/ceph/client/status.go index 7cae34c6e..1c462a236 100644 --- a/pkg/daemon/ceph/client/status.go +++ b/pkg/daemon/ceph/client/status.go @@ -48,12 +48,10 @@ type CephStatus struct { Quorum []int `json:"quorum"` QuorumNames []string `json:"quorum_names"` MonMap MonMap `json:"monmap"` - OsdMap struct { - OsdMap OsdMap `json:"osdmap"` - } `json:"osdmap"` - PgMap PgMap `json:"pgmap"` - MgrMap MgrMap `json:"mgrmap"` - Fsmap Fsmap `json:"fsmap"` + OsdMap OsdMap `json:"osdmap"` + PgMap PgMap `json:"pgmap"` + MgrMap MgrMap `json:"mgrmap"` + Fsmap Fsmap `json:"fsmap"` } type HealthStatus struct { @@ -72,6 +70,7 @@ type Summary struct { type MonMap struct { Epoch int `json:"epoch"` + NumMons int `josn:"num_mons"` FSID string `json:"fsid"` CreatedTime string `json:"created"` ModifiedTime string `json:"modified"` diff --git a/pkg/daemon/ceph/client/status_test.go b/pkg/daemon/ceph/client/status_test.go index 5da72e2ce..606352cf4 100644 --- a/pkg/daemon/ceph/client/status_test.go +++ b/pkg/daemon/ceph/client/status_test.go @@ -24,7 +24,7 @@ import ( const ( // this JSON was generated from `ceph status -f json`, using Ceph Luminous 12.1.3 - CephStatusResponseRaw = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}` + CephStatusResponseRaw = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}` ) var ( @@ -70,10 +70,10 @@ func TestStatusMarshal(t *testing.T) { assert.Equal(t, "10.3.0.45:6789/0", status.MonMap.Mons[0].Address) // verify some OSD map fields - assert.Equal(t, 2, status.OsdMap.OsdMap.NumOsd) - assert.Equal(t, 1, status.OsdMap.OsdMap.NumUpOsd) - assert.False(t, status.OsdMap.OsdMap.Full) - assert.True(t, status.OsdMap.OsdMap.NearFull) + assert.Equal(t, 2, status.OsdMap.NumOsd) + assert.Equal(t, 1, status.OsdMap.NumUpOsd) + assert.False(t, status.OsdMap.Full) + assert.True(t, status.OsdMap.NearFull) // verify some PG map fields assert.Equal(t, 200, status.PgMap.NumPgs) diff --git a/pkg/daemon/discover/discover.go b/pkg/daemon/discover/discover.go index 990151360..58e769fef 100644 --- a/pkg/daemon/discover/discover.go +++ b/pkg/daemon/discover/discover.go @@ -93,7 +93,7 @@ func Run(ctx context.Context, context *clusterd.Context, probeInterval time.Dura return err } - udevEvents := make(chan string) + udevEvents := make(chan struct{}) go udevBlockMonitor(udevEvents, udevEventPeriod) for { select { @@ -148,11 +148,11 @@ func matchUdevEvent(text string, matches, exclusions []string) (bool, error) { // Scans `udevadm monitor` output for block sub-system events. Each line of // output matching a set of substrings is sent to the provided channel. An event // is returned if it passes any matches tests, and passes all exclusion tests. -func rawUdevBlockMonitor(c chan string, matches, exclusions []string) { +func rawUdevBlockMonitor(c chan struct{}, matches, exclusions []string) { defer close(c) // stdbuf -oL performs line buffered output - cmd := exec.Command("stdbuf", "-oL", "udevadm", "monitor", "-u", "-k", "-s", "block") + cmd := exec.Command("stdbuf", "-oL", "udevadm", "monitor", "-u", "-s", "block") stdout, err := cmd.StdoutPipe() if err != nil { logger.Warningf("Cannot open udevadm stdout: %v", err) @@ -176,7 +176,7 @@ func rawUdevBlockMonitor(c chan string, matches, exclusions []string) { return } if match { - c <- text + c <- struct{}{} } } @@ -189,13 +189,13 @@ func rawUdevBlockMonitor(c chan string, matches, exclusions []string) { // Monitors udev for block device changes, and collapses these events such that // only one event is emitted per period in order to deal with flapping. -func udevBlockMonitor(c chan string, period time.Duration) { +func udevBlockMonitor(c chan struct{}, period time.Duration) { defer close(c) var udevFilter []string // return any add or remove events, but none that match device mapper // events. string matching is case-insensitive - events := make(chan string) + events := make(chan struct{}) // get discoverDaemonUdevBlacklist from the environment variable // if user doesn't provide any regex; generate the default regex @@ -211,16 +211,20 @@ func udevBlockMonitor(c chan string, period time.Duration) { []string{"(?i)add", "(?i)remove"}, udevFilter) + timeout := time.NewTimer(period) + defer timeout.Stop() for { - event, ok := <-events + _, ok := <-events if !ok { return } - timeout := time.NewTimer(period) + if !timeout.Stop() { + <-timeout.C + } + timeout.Reset(period) for { select { case <-timeout.C: - break case _, ok := <-events: if !ok { return @@ -229,7 +233,7 @@ func udevBlockMonitor(c chan string, period time.Duration) { } break } - c <- event + c <- struct{}{} } } diff --git a/tests/framework/clients/cluster.go b/tests/framework/clients/cluster.go index b812ad031..a0ecbcdc4 100644 --- a/tests/framework/clients/cluster.go +++ b/tests/framework/clients/cluster.go @@ -42,11 +42,11 @@ func IsClusterHealthy(testClient *TestClient, namespace string) (bool, error) { } // verify there are OSDs and they are all up/in - totalOSDs := status.OsdMap.OsdMap.NumOsd + totalOSDs := status.OsdMap.NumOsd if totalOSDs == 0 { return false, fmt.Errorf("no OSDs: %+v", status) } - if status.OsdMap.OsdMap.NumInOsd != totalOSDs || status.OsdMap.OsdMap.NumUpOsd != totalOSDs { + if status.OsdMap.NumInOsd != totalOSDs || status.OsdMap.NumUpOsd != totalOSDs { return false, fmt.Errorf("not all OSDs are up/in: %+v", status) } diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 0b8f3ab3c..a759b142c 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -866,8 +866,8 @@ func (k8sh *K8sHelper) IsPodInError(podNamePattern, namespace, reason, containin return false } -// GetPodHostID returns HostIP address of a pod -func (k8sh *K8sHelper) GetPodHostID(podNamePattern string, namespace string) (string, error) { +// GetPodHostIP returns HostIP address of a pod +func (k8sh *K8sHelper) GetPodHostIP(podNamePattern string, namespace string) (string, error) { ctx := context.TODO() listOpts := metav1.ListOptions{LabelSelector: "app=" + podNamePattern} podList, err := k8sh.Clientset.CoreV1().Pods(namespace).List(ctx, listOpts) @@ -1275,7 +1275,7 @@ func (k8sh *K8sHelper) getInternalRGWServiceURL(storeName string, namespace stri // GetRGWServiceURL returns URL of ceph RGW service in the cluster func (k8sh *K8sHelper) getExternalRGWServiceURL(storeName string, namespace string) (string, error) { - hostip, err := k8sh.GetPodHostID("rook-ceph-rgw", namespace) + hostip, err := k8sh.GetPodHostIP("rook-ceph-rgw", namespace) if err != nil { return "", fmt.Errorf("RGW pods not found. %+v", err) } diff --git a/tests/integration/ceph_base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go index 1c4ebab31..3625ac996 100644 --- a/tests/integration/ceph_base_deploy_test.go +++ b/tests/integration/ceph_base_deploy_test.go @@ -56,11 +56,14 @@ func checkIfRookClusterIsInstalled(s *suite.Suite, k8sh *utils.K8sHelper, opName func checkIfRookClusterIsHealthy(s *suite.Suite, testClient *clients.TestClient, clusterNamespace string) { logger.Infof("Testing cluster %s health", clusterNamespace) - var err error + var ( + err error + healthy bool + ) retryCount := 0 for retryCount < utils.RetryLoop { - healthy, err := clients.IsClusterHealthy(testClient, clusterNamespace) + healthy, err = clients.IsClusterHealthy(testClient, clusterNamespace) if healthy { logger.Infof("cluster %s is healthy", clusterNamespace) return