diff --git a/pkg/cephmgr/leader.go b/pkg/cephmgr/leader.go index 88e470acf..639aa395c 100644 --- a/pkg/cephmgr/leader.go +++ b/pkg/cephmgr/leader.go @@ -33,12 +33,13 @@ import ( // Interface implemented by a service that has been elected leader type cephLeader struct { - monLeader *mon.Leader - osdLeader *osd.Leader - mdsLeader *mds.Leader - rgwLeader *rgw.Leader - factory client.ConnectionFactory - adminSecret string + monLeader *mon.Leader + osdLeader *osd.Leader + mdsLeader *mds.Leader + rgwLeader *rgw.Leader + factory client.ConnectionFactory + adminSecret string + refreshInitialized bool } func newLeader(factory client.ConnectionFactory, adminSecret string) *cephLeader { @@ -68,14 +69,14 @@ func (c *cephLeader) RefreshKeys() []*clusterd.RefreshKey { return []*clusterd.RefreshKey{deviceChange, fileChange, objectChange} } -func getOSDsToRefresh(e *clusterd.RefreshEvent) *util.Set { +func getOSDsToRefresh(e *clusterd.RefreshEvent, refreshInitialized bool) *util.Set { osds := util.NewSet() osds.AddSet(e.NodesAdded) osds.AddSet(e.NodesChanged) osds.AddSet(e.NodesRemoved) - // Nothing changed in the event, so refresh osds on all nodes - if osds.Count() == 0 { + // Nothing changed in the event or this is the first refresh, so refresh osds on all nodes + if osds.Count() == 0 || !refreshInitialized { for nodeID := range e.Context.Inventory.Nodes { osds.Add(nodeID) } @@ -101,7 +102,7 @@ func (c *cephLeader) HandleRefresh(e *clusterd.RefreshEvent) { logger.Infof("ceph leader received refresh event") refreshMons := getRefreshMons(e) - osdsToRefresh := getOSDsToRefresh(e) + osdsToRefresh := getOSDsToRefresh(e, c.refreshInitialized) refreshFile := getRefreshFile(e) refreshObject := getRefreshObject(e) @@ -136,6 +137,7 @@ func (c *cephLeader) HandleRefresh(e *clusterd.RefreshEvent) { } } + c.refreshInitialized = true logger.Infof("ceph leader completed refresh") } diff --git a/pkg/cephmgr/leader_test.go b/pkg/cephmgr/leader_test.go index 0c6053a7b..0c317c8c8 100644 --- a/pkg/cephmgr/leader_test.go +++ b/pkg/cephmgr/leader_test.go @@ -76,6 +76,36 @@ func TestCephLeaders(t *testing.T) { assert.Equal(t, "mykey", etcdClient.GetValue("/rook/services/ceph/_secrets/admin")) } +func TestOSDRefresh(t *testing.T) { + factory := &testceph.MockConnectionFactory{Fsid: "myfsid", SecretKey: "mykey"} + leader := newLeader(factory, "") + + nodes := make(map[string]*inventory.NodeConfig) + nodes["a"] = &inventory.NodeConfig{PublicIP: "1.2.3.4"} + nodes["b"] = &inventory.NodeConfig{PublicIP: "2.2.3.4"} + + etcdClient := util.NewMockEtcdClient() + context := &clusterd.Context{EtcdClient: etcdClient, Inventory: &inventory.Config{Nodes: nodes}, ConfigDir: "/tmp"} + + // mock the agent responses that the deployments were successful to start mons and osds + etcdClient.WatcherResponses["/rook/_notify/a/monitor/status"] = "succeeded" + etcdClient.WatcherResponses["/rook/_notify/b/monitor/status"] = "succeeded" + etcdClient.WatcherResponses["/rook/_notify/a/osd/status"] = "succeeded" + etcdClient.WatcherResponses["/rook/_notify/b/osd/status"] = "succeeded" + + assert.Equal(t, "", etcdClient.GetValue("/rook/services/ceph/osd/desired/a/ready")) + assert.Equal(t, "", etcdClient.GetValue("/rook/services/ceph/osd/desired/b/ready")) + + // on first refresh we should trigger osds on all nodes, not just the node newly added + refresh := clusterd.NewRefreshEvent() + refresh.NodesAdded.Add("b") + refresh.Context = context + leader.HandleRefresh(refresh) + + assert.Equal(t, "1", etcdClient.GetValue("/rook/services/ceph/osd/desired/a/ready")) + assert.Equal(t, "1", etcdClient.GetValue("/rook/services/ceph/osd/desired/b/ready")) +} + func TestExtractDesiredDeviceNode(t *testing.T) { // valid path with node id node, err := extractNodeIDFromDesiredDevice("/rook/services/ceph/osd/desired/abc/device/sdb")