diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 72bb4dd85..885fbea12 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -151,9 +151,20 @@ func (c *Cluster) Start() error { logger.Warningf("failed to get storage nodes from namespace %s: %v", rookSystemNS, err) return err } + hostnameMap, err := k8sutil.GetNodeHostNames(c.context.Clientset) + if err != nil { + logger.Warningf("failed to get node hostnames: %v", err) + return err + } for nodeName := range allNodeDevices { + hostname, ok := hostnameMap[nodeName] + if !ok || nodeName == "" { + // fall back to the node name if no hostname is set + logger.Warningf("failed to get hostname for node %s. %+v", nodeName, err) + hostname = nodeName + } storageNode := rookalpha.Node{ - Name: nodeName, + Name: hostname, } c.Storage.Nodes = append(c.Storage.Nodes, storageNode) } diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 8f70b8bc6..a71dc41fa 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -49,7 +49,7 @@ const ( func (c *Cluster) makeJob(nodeName string, devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, storeConfig config.StoreConfig, metadataDevice, location string) (*batch.Job, error) { - podSpec, err := c.provisionPodTemplateSpec(devices, selection, resources, storeConfig, metadataDevice, location, v1.RestartPolicyOnFailure) + podSpec, err := c.provisionPodTemplateSpec(devices, selection, resources, storeConfig, metadataDevice, nodeName, location, v1.RestartPolicyOnFailure) if err != nil { return nil, err } @@ -112,13 +112,13 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se osdID := strconv.Itoa(osd.ID) tiniEnvVar := v1.EnvVar{Name: "TINI_SUBREAPER", Value: ""} envVars := []v1.EnvVar{ - nodeNameEnvVar(), + nodeNameEnvVar(nodeName), k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), tiniEnvVar, } envVars = append(envVars, k8sutil.ClusterDaemonEnvVars()...) - configEnvVars := append(c.getConfigEnvVars(storeConfig, dataDir, location), []v1.EnvVar{ + configEnvVars := append(c.getConfigEnvVars(storeConfig, dataDir, nodeName, location), []v1.EnvVar{ tiniEnvVar, {Name: "ROOK_OSD_ID", Value: osdID}, }...) @@ -250,7 +250,7 @@ func (c *Cluster) makeDeployment(nodeName string, devices []rookalpha.Device, se } func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, - storeConfig config.StoreConfig, metadataDevice, location string, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) { + storeConfig config.StoreConfig, metadataDevice, nodeName, location string, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) { volumes := opspec.PodVolumes(c.dataDirHostPath) @@ -278,7 +278,7 @@ func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection podSpec := v1.PodSpec{ ServiceAccountName: c.serviceAccount, - Containers: []v1.Container{c.provisionOSDContainer(devices, selection, resources, storeConfig, metadataDevice, location)}, + Containers: []v1.Container{c.provisionOSDContainer(devices, selection, resources, storeConfig, metadataDevice, nodeName, location)}, RestartPolicy: restart, Volumes: volumes, HostNetwork: c.HostNetwork, @@ -301,9 +301,9 @@ func (c *Cluster) provisionPodTemplateSpec(devices []rookalpha.Device, selection }, nil } -func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, location string) []v1.EnvVar { +func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, nodeName, location string) []v1.EnvVar { envVars := []v1.EnvVar{ - nodeNameEnvVar(), + nodeNameEnvVar(nodeName), {Name: "ROOK_CLUSTER_ID", Value: string(c.ownerRef.UID)}, k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), @@ -339,9 +339,9 @@ func (c *Cluster) getConfigEnvVars(storeConfig config.StoreConfig, dataDir, loca } func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, - storeConfig config.StoreConfig, metadataDevice, location string) v1.Container { + storeConfig config.StoreConfig, metadataDevice, nodeName, location string) v1.Container { - envVars := c.getConfigEnvVars(storeConfig, k8sutil.DataDir, location) + envVars := c.getConfigEnvVars(storeConfig, k8sutil.DataDir, nodeName, location) devMountNeeded := false privileged := false @@ -412,8 +412,8 @@ func (c *Cluster) provisionOSDContainer(devices []rookalpha.Device, selection ro } } -func nodeNameEnvVar() v1.EnvVar { - return v1.EnvVar{Name: "ROOK_NODE_NAME", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}} +func nodeNameEnvVar(name string) v1.EnvVar { + return v1.EnvVar{Name: "ROOK_NODE_NAME", Value: name} } func dataDevicesEnvVar(dataDevices string) v1.EnvVar { diff --git a/pkg/operator/ceph/cluster/osd/spec_test.go b/pkg/operator/ceph/cluster/osd/spec_test.go index 405b33b33..6b1eba264 100644 --- a/pkg/operator/ceph/cluster/osd/spec_test.go +++ b/pkg/operator/ceph/cluster/osd/spec_test.go @@ -36,7 +36,7 @@ import ( func TestPodContainer(t *testing.T) { cluster := &Cluster{Namespace: "myosd", Version: "23"} - c, err := cluster.provisionPodTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", v1.RestartPolicyAlways) + c, err := cluster.provisionPodTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "node", "", v1.RestartPolicyAlways) assert.NotNil(t, c) assert.Nil(t, err) assert.Equal(t, 1, len(c.Spec.Containers)) diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go index 39298c280..41ba24612 100644 --- a/pkg/operator/discover/discover.go +++ b/pkg/operator/discover/discover.go @@ -192,6 +192,15 @@ func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage, securityAcc // ListDevices lists all devices discovered on all nodes or specific node if node name is provided. func ListDevices(context *clusterd.Context, namespace, nodeName string) (map[string][]sys.LocalDisk, error) { + // convert the host name label to the k8s node name to look up the configmap with the devices + if len(nodeName) > 0 { + var err error + nodeName, err = k8sutil.GetNodeNameFromHostname(context.Clientset, nodeName) + if err != nil { + logger.Warningf("failed to get node name from hostname. %+v", err) + } + } + var devices map[string][]sys.LocalDisk listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, discoverDaemon.AppName)} // wait for device discovery configmaps diff --git a/pkg/operator/k8sutil/node.go b/pkg/operator/k8sutil/node.go index 26f9d55b2..9c933315d 100644 --- a/pkg/operator/k8sutil/node.go +++ b/pkg/operator/k8sutil/node.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" + "k8s.io/kubernetes/pkg/kubelet/apis" ) func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) { @@ -92,10 +93,16 @@ func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, p for _, node := range allNodes.Items { for _, rookNode := range rookNodes { - if rookNode.Name == node.Name { + hostname := node.Labels[apis.LabelHostname] + if len(hostname) == 0 { + // fall back to the node name if the hostname label is not set + hostname = node.Name + } + if rookNode.Name == hostname || rookNode.Name == node.Name { + rookNode.Name = hostname valid, err := ValidNode(node, placement) if err != nil { - logger.Warning("failed to validate node %s %v", node.Name, err) + logger.Warning("failed to validate node %s %v", rookNode.Name, err) } else if valid { validNodes = append(validNodes, rookNode) } @@ -105,3 +112,35 @@ func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, p } return validNodes } + +// GetNodeNameFromHostname returns the name of the node resource looked up by the hostname label +// Typically these will be the same name, but sometimes they are not such as when nodes have a longer +// dns name, but the hostname is short. +func GetNodeNameFromHostname(clientset kubernetes.Interface, hostName string) (string, error) { + options := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", apis.LabelHostname, hostName)} + nodes, err := clientset.CoreV1().Nodes().List(options) + if err != nil { + return hostName, err + } + + for _, node := range nodes.Items { + return node.Name, nil + } + return hostName, fmt.Errorf("node not found") +} + +// GetNodeHostNames returns the name of the node resource mapped to their hostname label. +// Typically these will be the same name, but sometimes they are not such as when nodes have a longer +// dns name, but the hostname is short. +func GetNodeHostNames(clientset kubernetes.Interface) (map[string]string, error) { + nodes, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + return nil, err + } + + nodeMap := map[string]string{} + for _, node := range nodes.Items { + nodeMap[node.Name] = node.Labels[apis.LabelHostname] + } + return nodeMap, nil +} diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index 41a08414e..a4a2c803e 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -55,6 +55,7 @@ type CephInstaller struct { hostPathToDelete string helmHelper *utils.HelmHelper k8sVersion string + changeHostnames bool T func() *testing.T } @@ -112,6 +113,11 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { return err } + if h.changeHostnames { + // give nodes a hostname that is different from its k8s node name to confirm that all the daemons will be initialized properly + h.k8shelper.ChangeHostnames() + } + rookOperator := h.Manifests.GetRookOperator(namespace) _, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...) @@ -456,6 +462,10 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(helmInstalled bool, systemNa logger.Infof("removing %s from node %s. err=%v", h.hostPathToDelete, node, err) } } + if h.changeHostnames { + // revert the hostname labels for the test + h.k8shelper.RestoreHostnames() + } } func (h *CephInstaller) cleanupDir(node, dir string) error { @@ -493,11 +503,12 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, rook panic("failed to get kubectl client :" + err.Error()) } h := &CephInstaller{ - Manifests: NewCephManifests(rookVersion), - k8shelper: k8shelp, - helmHelper: utils.NewHelmHelper(Env.Helm), - k8sVersion: version.String(), - T: t, + Manifests: NewCephManifests(rookVersion), + k8shelper: k8shelp, + helmHelper: utils.NewHelmHelper(Env.Helm), + k8sVersion: version.String(), + changeHostnames: rookVersion != Version0_8 && k8shelp.VersionAtLeast("v1.11.0"), + T: t, } flag.Parse() return h diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 54a459e4f..a51506528 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -40,6 +40,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" storagev1util "k8s.io/kubernetes/pkg/apis/storage/v1/util" + "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/pkg/util/version" ) @@ -59,6 +60,8 @@ const ( RetryInterval = 5 //TestMountPath is the path inside a test pod where storage is mounted TestMountPath = "/tmp/testrook" + //hostnameTestPrefix is a prefix added to the node hostname + hostnameTestPrefix = "testprefix-" ) //CreateK8sHelper creates a instance of k8sHelper @@ -1342,6 +1345,47 @@ func (k8sh *K8sHelper) GetExternalRGWServiceURL(storeName string, namespace stri return endpoint, err } +// ChangeHostnames modifies the node hostname label to run tests in an environment where the node name is different from the hostname label +func (k8sh *K8sHelper) ChangeHostnames() error { + nodes, err := k8sh.Clientset.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + return err + } + for _, node := range nodes.Items { + hostname := node.Labels[apis.LabelHostname] + if !strings.HasPrefix(hostname, hostnameTestPrefix) { + node.Labels[apis.LabelHostname] = hostnameTestPrefix + hostname + logger.Infof("changed hostname of node %s to %s", node.Name, node.Labels[apis.LabelHostname]) + _, err := k8sh.Clientset.CoreV1().Nodes().Update(&node) + if err != nil { + return err + } + } + } + + return nil +} + +// RestoreHostnames removes the test suffix from the node hostname labels +func (k8sh *K8sHelper) RestoreHostnames() ([]string, error) { + nodes, err := k8sh.Clientset.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + return nil, err + } + for _, node := range nodes.Items { + hostname := node.Labels[apis.LabelHostname] + if strings.HasPrefix(hostname, hostnameTestPrefix) { + node.Labels[apis.LabelHostname] = hostname[len(hostnameTestPrefix):] + logger.Infof("restoring hostname of node %s to %s", node.Name, node.Labels[apis.LabelHostname]) + _, err := k8sh.Clientset.CoreV1().Nodes().Update(&node) + if err != nil { + return nil, err + } + } + } + return nil, nil +} + //IsRookInstalled returns true is rook-ceph-mgr service is running(indicating rook is installed) func (k8sh *K8sHelper) IsRookInstalled(namespace string) bool { opts := metav1.GetOptions{}