Read node labels for provider topology

Signed-off-by: Jose A. Rivera <jarrpa@redhat.com>
This commit is contained in:
Jose A. Rivera committed 2019-05-14 13:18:00 -05:00
1 parent 5c5a62bd01
commit f3a1512983
7 files changed
+98 -28

No files matched your search

+3 -1
View File
@@ -71,7 +71,9 @@ For more details on the mons and when to choose a number other than `3`, see the
- `resources`: [resources configuration settings](#cluster-wide-resources-configuration-settings)
- `storage`: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes.
- `useAllNodes`: `true` or `false`, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values.
If individual nodes are specified under the `nodes` field below, then `useAllNodes` must be set to `false`.
If individual nodes are specified under the `nodes` field, then `useAllNodes` must be set to `false`.
- `topologyAware`: `true` or `false`, indicating whether Rook will look for and use topology/failure domain labels on Kubernetes nodes (e.g. "region" or "zone") as part of the CRUSH location for OSDs.
Node labels must follow the formatting of the failure domain [well-known labels](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/).
- `nodes`: Names of individual nodes in the cluster that should have their storage included in accordance with either the cluster level configuration specified above or any node specific overrides described in the next section below.
`useAllNodes` must be set to `false` to use specific nodes and their config.
See [node settings](#node-settings) below.
+3 -1
View File
@@ -4,7 +4,9 @@
## Notable Features
### <Storage Provider>
### Ceph
- Rook can now be configured to read "region" and "zone" labels on Kubernetes nodes and use that information as part of the CRUSH location for the OSDs.
## Breaking Changes
+6 -5
View File
@@ -27,11 +27,12 @@ import (
// ************************************************************************************
type StorageScopeSpec struct {
Nodes []Node `json:"nodes,omitempty"`
UseAllNodes bool `json:"useAllNodes,omitempty"`
NodeCount int `json:"nodeCount,omitempty"`
Location string `json:"location,omitempty"`
Config map[string]string `json:"config"`
Nodes []Node `json:"nodes,omitempty"`
UseAllNodes bool `json:"useAllNodes,omitempty"`
TopologyAware bool `json:"topologyAware,omitempty"`
NodeCount int `json:"nodeCount,omitempty"`
Location string `json:"location,omitempty"`
Config map[string]string `json:"config"`
Selection
}
+1 -1
View File
@@ -157,7 +157,7 @@ func (c *Cluster) Start() error {
logger.Debugf("storage nodes: %+v", c.DesiredStorage.Nodes)
}
// generally speaking, this finds nodes which are capable of running new osds
validNodes := k8sutil.GetValidNodes(c.DesiredStorage.Nodes, c.context.Clientset, c.placement)
validNodes := k8sutil.GetValidNodes(c.DesiredStorage, c.context.Clientset, c.placement)
// no valid node is ready to run an osd
if len(validNodes) == 0 {
+1 -1
View File
@@ -93,7 +93,7 @@ func (c *cluster) getClusterNodes() ([]rookalpha.Node, error) {
c.Spec.Storage.Nodes = append(c.Spec.Storage.Nodes, storageNode)
}
}
validNodes := k8sutil.GetValidNodes(c.Spec.Storage.Nodes, c.context.Clientset, edgefsv1beta1.GetTargetPlacement(c.Spec.Placement))
validNodes := k8sutil.GetValidNodes(c.Spec.Storage, c.context.Clientset, edgefsv1beta1.GetTargetPlacement(c.Spec.Placement))
c.Spec.Storage.Nodes = validNodes
return validNodes, nil
}
+38 -5
View File
@@ -19,6 +19,7 @@ package k8sutil
import (
"fmt"
"strings"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
v1 "k8s.io/api/core/v1"
@@ -28,6 +29,11 @@ import (
helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
)
var validTopologyLabelKeys = []string{
"failure-domain.beta.kubernetes.io",
"failure-domain.kubernetes.io",
}
// ValidNode returns true if the node (1) is schedulable, (2) meets Rook's placement terms, and
// (3) is ready. False otherwise.
func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) {
@@ -52,8 +58,8 @@ func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) {
// GetValidNodes returns all nodes that (1) are not cordoned, (2) meet Rook's placement terms, and
// (3) are ready.
func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, placement rookalpha.Placement) []rookalpha.Node {
matchingK8sNodes, err := GetKubernetesNodesMatchingRookNodes(rookNodes, clientset)
func GetValidNodes(rookStorage rookalpha.StorageScopeSpec, clientset kubernetes.Interface, placement rookalpha.Placement) []rookalpha.Node {
matchingK8sNodes, err := GetKubernetesNodesMatchingRookNodes(rookStorage.Nodes, clientset)
if err != nil {
// cannot list nodes, return empty nodes
logger.Errorf("failed to list nodes: %+v", err)
@@ -70,7 +76,7 @@ func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, p
}
}
return RookNodesMatchingKubernetesNodes(rookNodes, validK8sNodes)
return RookNodesMatchingKubernetesNodes(rookStorage, validK8sNodes)
}
// GetNodeNameFromHostname returns the name of the node resource looked up by the hostname label
@@ -224,12 +230,18 @@ func GetKubernetesNodesMatchingRookNodes(rookNodes []rookalpha.Node, clientset k
// RookNodesMatchingKubernetesNodes returns only the given Rook nodes which have a corresponding
// match in the list of Kubernetes nodes.
func RookNodesMatchingKubernetesNodes(rookNodes []rookalpha.Node, kubernetesNodes []v1.Node) []rookalpha.Node {
func RookNodesMatchingKubernetesNodes(rookStorage rookalpha.StorageScopeSpec, kubernetesNodes []v1.Node) []rookalpha.Node {
nodes := []rookalpha.Node{}
for _, kn := range kubernetesNodes {
for _, rn := range rookNodes {
for _, rn := range rookStorage.Nodes {
if rookNodeMatchesKubernetesNode(rn, kn) {
rn.Name = normalizeHostname(kn)
// modify the node Location if topology awareness enabled
if rookStorage.TopologyAware == true {
rn.Location = nodeTopologyLocation(kn, rn.Location)
}
nodes = append(nodes, rn)
}
}
@@ -237,6 +249,27 @@ func RookNodesMatchingKubernetesNodes(rookNodes []rookalpha.Node, kubernetesNode
return nodes
}
func nodeTopologyLocation(kubeNode v1.Node, location string) string {
nodeLabels := kubeNode.ObjectMeta.Labels
locations := []string{location}
// We're looking for node labels that match the following format:
// <validTopologyLabelKey>/<key>: <value>
// Where validTopologyLabelKey is an entry in the
// validTopologyLabelKeys list, key is a topology element (e.g. region,
// zone), and value is the name of the element (e.g. region1, zone2)
for label := range nodeLabels {
for _, key := range validTopologyLabelKeys {
if strings.Contains(label, key) {
keys := strings.Split(label, "/")
locations = append(locations, fmt.Sprintf("%s=%s", keys[1], nodeLabels[label]))
}
}
}
return strings.Join(locations, " ")
}
// NodeIsInRookNodeList will return true if the target node is found in a given list of Rook nodes.
func NodeIsInRookNodeList(targetNodeName string, rookNodes []rookalpha.Node) bool {
for _, rn := range rookNodes {
+46 -14
View File
@@ -50,12 +50,14 @@ func TestValidNode(t *testing.T) {
nodeA := "nodeA"
nodeB := "nodeB"
nodes := []rookalpha.Node{
{
Name: nodeA,
},
{
Name: nodeB,
storage := rookalpha.StorageScopeSpec{
Nodes: []rookalpha.Node{
{
Name: nodeA,
},
{
Name: nodeB,
},
},
}
var placement rookalpha.Placement
@@ -66,7 +68,7 @@ func TestValidNode(t *testing.T) {
assert.Nil(t, nodeErr)
nodeErr = createNode(nodeB, v1.NodeNetworkUnavailable, clientset)
assert.Nil(t, nodeErr)
validNodes := GetValidNodes(nodes, clientset, placement)
validNodes := GetValidNodes(storage, clientset, placement)
assert.Equal(t, len(validNodes), 1)
}
@@ -265,16 +267,18 @@ func TestRookNodesMatchingKubernetesNodes(t *testing.T) {
k8sNodes := []v1.Node{n0, n1, n2}
// no rook nodes specified for input
rookNodes := []rookalpha.Node{}
retNodes := RookNodesMatchingKubernetesNodes(rookNodes, k8sNodes)
rookStorage := rookalpha.StorageScopeSpec{
Nodes: []rookalpha.Node{},
}
retNodes := RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes)
assert.Len(t, retNodes, 0)
// all rook nodes specified
rookNodes = []rookalpha.Node{
rookStorage.Nodes = []rookalpha.Node{
{Name: "node0"},
{Name: "node1"},
{Name: "node2"}}
retNodes = RookNodesMatchingKubernetesNodes(rookNodes, k8sNodes)
retNodes = RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes)
assert.Len(t, retNodes, 3)
// this should return nodes named by hostname if that is available
assert.Contains(t, retNodes, rookalpha.Node{Name: "node0-hostname"})
@@ -282,18 +286,46 @@ func TestRookNodesMatchingKubernetesNodes(t *testing.T) {
assert.Contains(t, retNodes, rookalpha.Node{Name: "node2"})
// more rook nodes specified than exist
rookNodes = []rookalpha.Node{
rookStorage.Nodes = []rookalpha.Node{
{Name: "node0-hostname"},
{Name: "node2"},
{Name: "node5"}}
retNodes = RookNodesMatchingKubernetesNodes(rookNodes, k8sNodes)
retNodes = RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes)
assert.Len(t, retNodes, 2)
assert.Contains(t, retNodes, rookalpha.Node{Name: "node0-hostname"})
assert.Contains(t, retNodes, rookalpha.Node{Name: "node2"})
// no k8s nodes specified
retNodes = RookNodesMatchingKubernetesNodes(rookNodes, []v1.Node{})
retNodes = RookNodesMatchingKubernetesNodes(rookStorage, []v1.Node{})
assert.Len(t, retNodes, 0)
// topology awareness
rookStorage.Nodes = []rookalpha.Node{
{Name: "node0"},
{Name: "node1"},
{Name: "node2"}}
rookStorage.TopologyAware = true
k8sNodes[0].ObjectMeta.Labels = map[string]string{
"failure-domain.beta.kubernetes.io/region": "region1",
"failure-domain.beta.kubernetes.io/zone": "zone1",
}
k8sNodes[1].ObjectMeta.Labels = map[string]string{
"failure-domain.beta.kubernetes.io/region": "region1",
"failure-domain.beta.kubernetes.io/zone": "zone2",
"foo": "bar",
}
k8sNodes[2].ObjectMeta.Labels = map[string]string{}
retNodes = RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes)
assert.Contains(t, retNodes[0].Location, "region=region1")
assert.Contains(t, retNodes[0].Location, "zone=zone1")
assert.Contains(t, retNodes[1].Location, "region=region1")
assert.Contains(t, retNodes[1].Location, "zone=zone2")
assert.NotContains(t, retNodes[1].Location, "foo=bar")
assert.NotContains(t, retNodes[2].Location, "region")
assert.NotContains(t, retNodes[2].Location, "zone")
}
func TestNodeIsInRookList(t *testing.T) {