osd: use the node hostname labels instead of node names

Signed-off-by: travisn <tnielsen@redhat.com>

pkg/operator/ceph/cluster/osd/osd.go

@@ -151,9 +151,20 @@ func (c *Cluster) Start() error {
 			logger.Warningf("failed to get storage nodes from namespace %s: %v", rookSystemNS, err)
 			return err
 		}
+		hostnameMap, err := k8sutil.GetNodeHostNames(c.context.Clientset)
+		if err != nil {
+			logger.Warningf("failed to get node hostnames: %v", err)
+			return err
+		}
 		for nodeName := range allNodeDevices {
+			hostname, ok := hostnameMap[nodeName]
+			if !ok || nodeName == "" {
+				// fall back to the node name if no hostname is set
+				logger.Warningf("failed to get hostname for node %s. %+v", nodeName, err)
+				hostname = nodeName
+			}
 			storageNode := rookalpha.Node{
-				Name: nodeName,
+				Name: hostname,
 			}
 			c.Storage.Nodes = append(c.Storage.Nodes, storageNode)
 		}

pkg/operator/ceph/cluster/osd/spec.go

@@ -49,7 +49,7 @@ const (
 func (c *Cluster) makeJob(nodeName string, devices []rookalpha.Device,
 	selection rookalpha.Selection, resources v1.ResourceRequirements, storeConfig config.StoreConfig, metadataDevice, location string) (*batch.Job, error) {
 
-	podSpec, err := c.provisionPodTemplateSpec(devices, selection, resources, storeConfig, metadataDevice, location, v1.RestartPolicyOnFailure)
+	podSpec, err := c.provisionPodTemplateSpec(devices, selection, resources, storeConfig, metadataDevice, nodeName, location, v1.RestartPolicyOnFailure)
 	if err != nil {
 		return nil, err
 	}
@@ -112,13 +112,13 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
 	osdID := strconv.Itoa(osd.ID)
 	tiniEnvVar := v1.EnvVar{Name: "TINI_SUBREAPER", Value: ""}
 	envVars := []v1.EnvVar{
-		nodeNameEnvVar(),
+		nodeNameEnvVar(nodeName),
 		k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
 		k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
 		tiniEnvVar,
 	}
 	envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...)
-	configEnvVars := append(c.getConfigEnvVars(storeConfig, dataDir, location), []v1.EnvVar{
+	configEnvVars := append(c.getConfigEnvVars(storeConfig, dataDir, nodeName, location), []v1.EnvVar{
 		tiniEnvVar,
 		{Name: "ROOK_OSD_ID", Value: osdID},
 	}...)
@@ -250,7 +250,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se
 }
 
 func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements,
-	storeConfig config.StoreConfig, metadataDevice, location string, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) {
+	storeConfig config.StoreConfig, metadataDevice, nodeName, location string, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) {
 
 	volumes := opspec.PodVolumes(c.dataDirHostPath)
 
@@ -278,7 +278,7 @@ func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection
 
 	podSpec := v1.PodSpec{
 		ServiceAccountName: c.serviceAccount,
-		Containers:         []v1.Container{c.provisionOSDContainer(devices, selection, resources, storeConfig, metadataDevice, location)},
+		Containers:         []v1.Container{c.provisionOSDContainer(devices, selection, resources, storeConfig, metadataDevice, nodeName, location)},
 		RestartPolicy:      restart,
 		Volumes:            volumes,
 		HostNetwork:        c.HostNetwork,
@@ -301,9 +301,9 @@ func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection
 	}, nil
 }
 
-func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, location string) []v1.EnvVar {
+func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, nodeName, location string) []v1.EnvVar {
 	envVars := []v1.EnvVar{
-		nodeNameEnvVar(),
+		nodeNameEnvVar(nodeName),
 		{Name: "ROOK_CLUSTER_ID", Value: string(c.ownerRef.UID)},
 		k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
 		k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
@@ -339,9 +339,9 @@ func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, loca
 }
 
 func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements,
-	storeConfig config.StoreConfig, metadataDevice, location string) v1.Container {
+	storeConfig config.StoreConfig, metadataDevice, nodeName, location string) v1.Container {
 
-	envVars := c.getConfigEnvVars(storeConfig, k8sutil.DataDir, location)
+	envVars := c.getConfigEnvVars(storeConfig, k8sutil.DataDir, nodeName, location)
 	devMountNeeded := false
 	privileged := false
 
@@ -412,8 +412,8 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro
 	}
 }
 
-func nodeNameEnvVar() v1.EnvVar {
-	return v1.EnvVar{Name: "ROOK_NODE_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}}
+func nodeNameEnvVar(name string) v1.EnvVar {
+	return v1.EnvVar{Name: "ROOK_NODE_NAME", Value: name}
 }
 
 func dataDevicesEnvVar(dataDevices string) v1.EnvVar {

pkg/operator/ceph/cluster/osd/spec_test.go

@@ -36,7 +36,7 @@ import (
 
 func TestPodContainer(t *testing.T) {
 	cluster := &Cluster{Namespace: "myosd", Version: "23"}
-	c, err := cluster.provisionPodTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", v1.RestartPolicyAlways)
+	c, err := cluster.provisionPodTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "node", "", v1.RestartPolicyAlways)
 	assert.NotNil(t, c)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(c.Spec.Containers))

pkg/operator/discover/discover.go

@@ -192,6 +192,15 @@ func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage, securityAcc
 
 // ListDevices lists all devices discovered on all nodes or specific node if node name is provided.
 func ListDevices(context *clusterd.Context, namespace, nodeName string) (map[string][]sys.LocalDisk, error) {
+	// convert the host name label to the k8s node name to look up the configmap  with the devices
+	if len(nodeName) > 0 {
+		var err error
+		nodeName, err = k8sutil.GetNodeNameFromHostname(context.Clientset, nodeName)
+		if err != nil {
+			logger.Warningf("failed to get node name from hostname. %+v", err)
+		}
+	}
+
 	var devices map[string][]sys.LocalDisk
 	listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, discoverDaemon.AppName)}
 	// wait for device discovery configmaps

pkg/operator/k8sutil/node.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 	helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
+	"k8s.io/kubernetes/pkg/kubelet/apis"
 )
 
 func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) {
@@ -92,10 +93,16 @@ func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, p
 
 	for _, node := range allNodes.Items {
 		for _, rookNode := range rookNodes {
-			if rookNode.Name == node.Name {
+			hostname := node.Labels[apis.LabelHostname]
+			if len(hostname) == 0 {
+				// fall back to the node name if the hostname label is not set
+				hostname = node.Name
+			}
+			if rookNode.Name == hostname || rookNode.Name == node.Name {
+				rookNode.Name = hostname
 				valid, err := ValidNode(node, placement)
 				if err != nil {
-					logger.Warning("failed to validate node %s %v", node.Name, err)
+					logger.Warning("failed to validate node %s %v", rookNode.Name, err)
 				} else if valid {
 					validNodes = append(validNodes, rookNode)
 				}
@@ -105,3 +112,35 @@ func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, p
 	}
 	return validNodes
 }
+
+// GetNodeNameFromHostname returns the name of the node resource looked up by the hostname label
+// Typically these will be the same name, but sometimes they are not such as when nodes have a longer
+// dns name, but the hostname is short.
+func GetNodeNameFromHostname(clientset kubernetes.Interface, hostName string) (string, error) {
+	options := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", apis.LabelHostname, hostName)}
+	nodes, err := clientset.CoreV1().Nodes().List(options)
+	if err != nil {
+		return hostName, err
+	}
+
+	for _, node := range nodes.Items {
+		return node.Name, nil
+	}
+	return hostName, fmt.Errorf("node not found")
+}
+
+// GetNodeHostNames returns the name of the node resource mapped to their hostname label.
+// Typically these will be the same name, but sometimes they are not such as when nodes have a longer
+// dns name, but the hostname is short.
+func GetNodeHostNames(clientset kubernetes.Interface) (map[string]string, error) {
+	nodes, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	nodeMap := map[string]string{}
+	for _, node := range nodes.Items {
+		nodeMap[node.Name] = node.Labels[apis.LabelHostname]
+	}
+	return nodeMap, nil
+}

tests/framework/installer/ceph_installer.go

@@ -55,6 +55,7 @@ type CephInstaller struct {
 	hostPathToDelete string
 	helmHelper       *utils.HelmHelper
 	k8sVersion       string
+	changeHostnames  bool
 	T                func() *testing.T
 }
 
@@ -112,6 +113,11 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) {
 		return err
 	}
 
+	if h.changeHostnames {
+		// give nodes a hostname that is different from its k8s node name to confirm that all the daemons will be initialized properly
+		h.k8shelper.ChangeHostnames()
+	}
+
 	rookOperator := h.Manifests.GetRookOperator(namespace)
 
 	_, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...)
@@ -456,6 +462,10 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
 			logger.Infof("removing %s from node %s. err=%v", h.hostPathToDelete, node, err)
 		}
 	}
+	if h.changeHostnames {
+		// revert the hostname labels for the test
+		h.k8shelper.RestoreHostnames()
+	}
 }
 
 func (h *CephInstaller) cleanupDir(node, dir string) error {
@@ -493,11 +503,12 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rook
 		panic("failed to get kubectl client :" + err.Error())
 	}
 	h := &CephInstaller{
-		Manifests:  NewCephManifests(rookVersion),
-		k8shelper:  k8shelp,
-		helmHelper: utils.NewHelmHelper(Env.Helm),
-		k8sVersion: version.String(),
-		T:          t,
+		Manifests:       NewCephManifests(rookVersion),
+		k8shelper:       k8shelp,
+		helmHelper:      utils.NewHelmHelper(Env.Helm),
+		k8sVersion:      version.String(),
+		changeHostnames: rookVersion != Version0_8 && k8shelp.VersionAtLeast("v1.11.0"),
+		T:               t,
 	}
 	flag.Parse()
 	return h

tests/framework/utils/k8s_helper.go

@@ -40,6 +40,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	storagev1util "k8s.io/kubernetes/pkg/apis/storage/v1/util"
+	"k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/pkg/util/version"
 )
 
@@ -59,6 +60,8 @@ const (
 	RetryInterval = 5
 	//TestMountPath is the path inside a test pod where storage is mounted
 	TestMountPath = "/tmp/testrook"
+	//hostnameTestPrefix is a prefix added to the node hostname
+	hostnameTestPrefix = "testprefix-"
 )
 
 //CreateK8sHelper creates a instance of k8sHelper
@@ -1342,6 +1345,47 @@ func (k8sh *K8sHelper) GetExternalRGWServiceURL(storeName string, namespace stri
 	return endpoint, err
 }
 
+// ChangeHostnames modifies the node hostname label to run tests in an environment where the node name is different from the hostname label
+func (k8sh *K8sHelper) ChangeHostnames() error {
+	nodes, err := k8sh.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, node := range nodes.Items {
+		hostname := node.Labels[apis.LabelHostname]
+		if !strings.HasPrefix(hostname, hostnameTestPrefix) {
+			node.Labels[apis.LabelHostname] = hostnameTestPrefix + hostname
+			logger.Infof("changed hostname of node %s to %s", node.Name, node.Labels[apis.LabelHostname])
+			_, err := k8sh.Clientset.CoreV1().Nodes().Update(&node)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// RestoreHostnames removes the test suffix from the node hostname labels
+func (k8sh *K8sHelper) RestoreHostnames() ([]string, error) {
+	nodes, err := k8sh.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	for _, node := range nodes.Items {
+		hostname := node.Labels[apis.LabelHostname]
+		if strings.HasPrefix(hostname, hostnameTestPrefix) {
+			node.Labels[apis.LabelHostname] = hostname[len(hostnameTestPrefix):]
+			logger.Infof("restoring hostname of node %s to %s", node.Name, node.Labels[apis.LabelHostname])
+			_, err := k8sh.Clientset.CoreV1().Nodes().Update(&node)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return nil, nil
+}
+
 //IsRookInstalled returns true is rook-ceph-mgr service is running(indicating rook is installed)
 func (k8sh *K8sHelper) IsRookInstalled(namespace string) bool {
 	opts := metav1.GetOptions{}