Set ip address for monitor

pkg/castled/cmd/castled.go

@@ -74,9 +74,9 @@ func joinCluster(cmd *cobra.Command, args []string) error {
 	// Write desired state for this machine to etcd
 	baseKey := path.Join(orchestrator.DesiredNodesKey, nodeID)
 	properties := map[string]string{
-		"privateIPv4": privateIPv4,
-		"devices":     devices,
-		"forceFormat": strconv.FormatBool(forceFormat),
+		castled.PrivateIPv4Value: privateIPv4,
+		castled.DevicesValue:     devices,
+		castled.ForceFormatValue: strconv.FormatBool(forceFormat),
 	}
 	if err := orchestrator.StoreEtcdProperties(etcdClient, baseKey, properties); err != nil {
 		return err

pkg/castled/mon.go

@@ -11,6 +11,7 @@ import (
 )
 
 const (
+	PrivateIPv4Value       = "privateIPv4"
 	monitorKey             = "ceph-mon"
 	osdKey                 = "ceph-osd"
 	monitorKeyringTemplate = `

pkg/castled/monleader.go

@@ -297,11 +297,17 @@ func (m *monLeader) chooseMonitorNodes(context *orchestrator.ClusterContext) (ma
 	monitorNum := 0
 	var settings = make(map[string]string)
 	for nodeID, node := range context.Inventory.Nodes {
+		ipaddress, err := getDesiredNodeIPAddress(context, nodeID)
+		if err != nil {
+			log.Printf("failed to discover desired ip address for node %s. %v", nodeID, err)
+			return nil, err
+		}
+
 		// Store the monitor id and connection info
 		monitorID := strconv.FormatInt(int64(monitorNum), 10)
 		port := "6790"
 		settings[path.Join(nodeID, "id")] = monitorID
-		settings[path.Join(nodeID, "ipaddress")] = node.IPAddress
+		settings[path.Join(nodeID, "ipaddress")] = ipaddress
 		settings[path.Join(nodeID, "port")] = port
 
 		monitor := &CephMonitorConfig{Name: monitorID, Endpoint: fmt.Sprintf("%s:%s", node.IPAddress, port)}
@@ -320,6 +326,16 @@ func (m *monLeader) chooseMonitorNodes(context *orchestrator.ClusterContext) (ma
 	return monitors, nil
 }
 
+func getDesiredNodeIPAddress(context *orchestrator.ClusterContext, nodeID string) (string, error) {
+	key := path.Join(orchestrator.DesiredNodesKey, nodeID, PrivateIPv4Value)
+	resp, err := context.EtcdClient.Get(ctx.Background(), key, nil)
+	if err != nil {
+		return "", err
+	}
+
+	return resp.Node.Value, nil
+}
+
 // Calculate the number of monitors that should be deployed
 func calculateMonitorCount(nodeCount int) int {
 	if nodeCount > 100 {

pkg/castled/osd.go

@@ -20,6 +20,8 @@ import (
 )
 
 const (
+	DevicesValue                = "devices"
+	ForceFormatValue            = "forceFormat"
 	bootstrapOSDKeyringTemplate = `
 [client.bootstrap-osd]
 	key = %s