Read node labels for provider topology
Signed-off-by: Jose A. Rivera <jarrpa@redhat.com>
This commit is contained in:
1 parent
5c5a62bd01
commit
f3a1512983
7 files changed
+98
-28
No files matched your search
@@ -71,7 +71,9 @@ For more details on the mons and when to choose a number other than `3`, see the
|
||||
- `resources`: [resources configuration settings](#cluster-wide-resources-configuration-settings)
|
||||
- `storage`: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes.
|
||||
- `useAllNodes`: `true` or `false`, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values.
|
||||
If individual nodes are specified under the `nodes` field below, then `useAllNodes` must be set to `false`.
|
||||
If individual nodes are specified under the `nodes` field, then `useAllNodes` must be set to `false`.
|
||||
- `topologyAware`: `true` or `false`, indicating whether Rook will look for and use topology/failure domain labels on Kubernetes nodes (e.g. "region" or "zone") as part of the CRUSH location for OSDs.
|
||||
Node labels must follow the formatting of the failure domain [well-known labels](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/).
|
||||
- `nodes`: Names of individual nodes in the cluster that should have their storage included in accordance with either the cluster level configuration specified above or any node specific overrides described in the next section below.
|
||||
`useAllNodes` must be set to `false` to use specific nodes and their config.
|
||||
See [node settings](#node-settings) below.
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
|
||||
## Notable Features
|
||||
|
||||
### <Storage Provider>
|
||||
### Ceph
|
||||
|
||||
- Rook can now be configured to read "region" and "zone" labels on Kubernetes nodes and use that information as part of the CRUSH location for the OSDs.
|
||||
|
||||
## Breaking Changes
|
||||
|
||||
|
||||
@@ -27,11 +27,12 @@ import (
|
||||
// ************************************************************************************
|
||||
|
||||
type StorageScopeSpec struct {
|
||||
Nodes []Node `json:"nodes,omitempty"`
|
||||
UseAllNodes bool `json:"useAllNodes,omitempty"`
|
||||
NodeCount int `json:"nodeCount,omitempty"`
|
||||
Location string `json:"location,omitempty"`
|
||||
Config map[string]string `json:"config"`
|
||||
Nodes []Node `json:"nodes,omitempty"`
|
||||
UseAllNodes bool `json:"useAllNodes,omitempty"`
|
||||
TopologyAware bool `json:"topologyAware,omitempty"`
|
||||
NodeCount int `json:"nodeCount,omitempty"`
|
||||
Location string `json:"location,omitempty"`
|
||||
Config map[string]string `json:"config"`
|
||||
Selection
|
||||
}
|
||||
|
||||
|
||||
@@ -157,7 +157,7 @@ func (c *Cluster) Start() error {
|
||||
logger.Debugf("storage nodes: %+v", c.DesiredStorage.Nodes)
|
||||
}
|
||||
// generally speaking, this finds nodes which are capable of running new osds
|
||||
validNodes := k8sutil.GetValidNodes(c.DesiredStorage.Nodes, c.context.Clientset, c.placement)
|
||||
validNodes := k8sutil.GetValidNodes(c.DesiredStorage, c.context.Clientset, c.placement)
|
||||
|
||||
// no valid node is ready to run an osd
|
||||
if len(validNodes) == 0 {
|
||||
|
||||
@@ -93,7 +93,7 @@ func (c *cluster) getClusterNodes() ([]rookalpha.Node, error) {
|
||||
c.Spec.Storage.Nodes = append(c.Spec.Storage.Nodes, storageNode)
|
||||
}
|
||||
}
|
||||
validNodes := k8sutil.GetValidNodes(c.Spec.Storage.Nodes, c.context.Clientset, edgefsv1beta1.GetTargetPlacement(c.Spec.Placement))
|
||||
validNodes := k8sutil.GetValidNodes(c.Spec.Storage, c.context.Clientset, edgefsv1beta1.GetTargetPlacement(c.Spec.Placement))
|
||||
c.Spec.Storage.Nodes = validNodes
|
||||
return validNodes, nil
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ package k8sutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
@@ -28,6 +29,11 @@ import (
|
||||
helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
|
||||
)
|
||||
|
||||
var validTopologyLabelKeys = []string{
|
||||
"failure-domain.beta.kubernetes.io",
|
||||
"failure-domain.kubernetes.io",
|
||||
}
|
||||
|
||||
// ValidNode returns true if the node (1) is schedulable, (2) meets Rook's placement terms, and
|
||||
// (3) is ready. False otherwise.
|
||||
func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) {
|
||||
@@ -52,8 +58,8 @@ func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) {
|
||||
|
||||
// GetValidNodes returns all nodes that (1) are not cordoned, (2) meet Rook's placement terms, and
|
||||
// (3) are ready.
|
||||
func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, placement rookalpha.Placement) []rookalpha.Node {
|
||||
matchingK8sNodes, err := GetKubernetesNodesMatchingRookNodes(rookNodes, clientset)
|
||||
func GetValidNodes(rookStorage rookalpha.StorageScopeSpec, clientset kubernetes.Interface, placement rookalpha.Placement) []rookalpha.Node {
|
||||
matchingK8sNodes, err := GetKubernetesNodesMatchingRookNodes(rookStorage.Nodes, clientset)
|
||||
if err != nil {
|
||||
// cannot list nodes, return empty nodes
|
||||
logger.Errorf("failed to list nodes: %+v", err)
|
||||
@@ -70,7 +76,7 @@ func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, p
|
||||
}
|
||||
}
|
||||
|
||||
return RookNodesMatchingKubernetesNodes(rookNodes, validK8sNodes)
|
||||
return RookNodesMatchingKubernetesNodes(rookStorage, validK8sNodes)
|
||||
}
|
||||
|
||||
// GetNodeNameFromHostname returns the name of the node resource looked up by the hostname label
|
||||
@@ -224,12 +230,18 @@ func GetKubernetesNodesMatchingRookNodes(rookNodes []rookalpha.Node, clientset k
|
||||
|
||||
// RookNodesMatchingKubernetesNodes returns only the given Rook nodes which have a corresponding
|
||||
// match in the list of Kubernetes nodes.
|
||||
func RookNodesMatchingKubernetesNodes(rookNodes []rookalpha.Node, kubernetesNodes []v1.Node) []rookalpha.Node {
|
||||
func RookNodesMatchingKubernetesNodes(rookStorage rookalpha.StorageScopeSpec, kubernetesNodes []v1.Node) []rookalpha.Node {
|
||||
nodes := []rookalpha.Node{}
|
||||
for _, kn := range kubernetesNodes {
|
||||
for _, rn := range rookNodes {
|
||||
for _, rn := range rookStorage.Nodes {
|
||||
if rookNodeMatchesKubernetesNode(rn, kn) {
|
||||
rn.Name = normalizeHostname(kn)
|
||||
|
||||
// modify the node Location if topology awareness enabled
|
||||
if rookStorage.TopologyAware == true {
|
||||
rn.Location = nodeTopologyLocation(kn, rn.Location)
|
||||
}
|
||||
|
||||
nodes = append(nodes, rn)
|
||||
}
|
||||
}
|
||||
@@ -237,6 +249,27 @@ func RookNodesMatchingKubernetesNodes(rookNodes []rookalpha.Node, kubernetesNode
|
||||
return nodes
|
||||
}
|
||||
|
||||
func nodeTopologyLocation(kubeNode v1.Node, location string) string {
|
||||
nodeLabels := kubeNode.ObjectMeta.Labels
|
||||
locations := []string{location}
|
||||
|
||||
// We're looking for node labels that match the following format:
|
||||
// <validTopologyLabelKey>/<key>: <value>
|
||||
// Where validTopologyLabelKey is an entry in the
|
||||
// validTopologyLabelKeys list, key is a topology element (e.g. region,
|
||||
// zone), and value is the name of the element (e.g. region1, zone2)
|
||||
for label := range nodeLabels {
|
||||
for _, key := range validTopologyLabelKeys {
|
||||
if strings.Contains(label, key) {
|
||||
keys := strings.Split(label, "/")
|
||||
locations = append(locations, fmt.Sprintf("%s=%s", keys[1], nodeLabels[label]))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(locations, " ")
|
||||
}
|
||||
|
||||
// NodeIsInRookNodeList will return true if the target node is found in a given list of Rook nodes.
|
||||
func NodeIsInRookNodeList(targetNodeName string, rookNodes []rookalpha.Node) bool {
|
||||
for _, rn := range rookNodes {
|
||||
|
||||
@@ -50,12 +50,14 @@ func TestValidNode(t *testing.T) {
|
||||
nodeA := "nodeA"
|
||||
nodeB := "nodeB"
|
||||
|
||||
nodes := []rookalpha.Node{
|
||||
{
|
||||
Name: nodeA,
|
||||
},
|
||||
{
|
||||
Name: nodeB,
|
||||
storage := rookalpha.StorageScopeSpec{
|
||||
Nodes: []rookalpha.Node{
|
||||
{
|
||||
Name: nodeA,
|
||||
},
|
||||
{
|
||||
Name: nodeB,
|
||||
},
|
||||
},
|
||||
}
|
||||
var placement rookalpha.Placement
|
||||
@@ -66,7 +68,7 @@ func TestValidNode(t *testing.T) {
|
||||
assert.Nil(t, nodeErr)
|
||||
nodeErr = createNode(nodeB, v1.NodeNetworkUnavailable, clientset)
|
||||
assert.Nil(t, nodeErr)
|
||||
validNodes := GetValidNodes(nodes, clientset, placement)
|
||||
validNodes := GetValidNodes(storage, clientset, placement)
|
||||
assert.Equal(t, len(validNodes), 1)
|
||||
}
|
||||
|
||||
@@ -265,16 +267,18 @@ func TestRookNodesMatchingKubernetesNodes(t *testing.T) {
|
||||
k8sNodes := []v1.Node{n0, n1, n2}
|
||||
|
||||
// no rook nodes specified for input
|
||||
rookNodes := []rookalpha.Node{}
|
||||
retNodes := RookNodesMatchingKubernetesNodes(rookNodes, k8sNodes)
|
||||
rookStorage := rookalpha.StorageScopeSpec{
|
||||
Nodes: []rookalpha.Node{},
|
||||
}
|
||||
retNodes := RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes)
|
||||
assert.Len(t, retNodes, 0)
|
||||
|
||||
// all rook nodes specified
|
||||
rookNodes = []rookalpha.Node{
|
||||
rookStorage.Nodes = []rookalpha.Node{
|
||||
{Name: "node0"},
|
||||
{Name: "node1"},
|
||||
{Name: "node2"}}
|
||||
retNodes = RookNodesMatchingKubernetesNodes(rookNodes, k8sNodes)
|
||||
retNodes = RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes)
|
||||
assert.Len(t, retNodes, 3)
|
||||
// this should return nodes named by hostname if that is available
|
||||
assert.Contains(t, retNodes, rookalpha.Node{Name: "node0-hostname"})
|
||||
@@ -282,18 +286,46 @@ func TestRookNodesMatchingKubernetesNodes(t *testing.T) {
|
||||
assert.Contains(t, retNodes, rookalpha.Node{Name: "node2"})
|
||||
|
||||
// more rook nodes specified than exist
|
||||
rookNodes = []rookalpha.Node{
|
||||
rookStorage.Nodes = []rookalpha.Node{
|
||||
{Name: "node0-hostname"},
|
||||
{Name: "node2"},
|
||||
{Name: "node5"}}
|
||||
retNodes = RookNodesMatchingKubernetesNodes(rookNodes, k8sNodes)
|
||||
retNodes = RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes)
|
||||
assert.Len(t, retNodes, 2)
|
||||
assert.Contains(t, retNodes, rookalpha.Node{Name: "node0-hostname"})
|
||||
assert.Contains(t, retNodes, rookalpha.Node{Name: "node2"})
|
||||
|
||||
// no k8s nodes specified
|
||||
retNodes = RookNodesMatchingKubernetesNodes(rookNodes, []v1.Node{})
|
||||
retNodes = RookNodesMatchingKubernetesNodes(rookStorage, []v1.Node{})
|
||||
assert.Len(t, retNodes, 0)
|
||||
|
||||
// topology awareness
|
||||
rookStorage.Nodes = []rookalpha.Node{
|
||||
{Name: "node0"},
|
||||
{Name: "node1"},
|
||||
{Name: "node2"}}
|
||||
rookStorage.TopologyAware = true
|
||||
|
||||
k8sNodes[0].ObjectMeta.Labels = map[string]string{
|
||||
"failure-domain.beta.kubernetes.io/region": "region1",
|
||||
"failure-domain.beta.kubernetes.io/zone": "zone1",
|
||||
}
|
||||
k8sNodes[1].ObjectMeta.Labels = map[string]string{
|
||||
"failure-domain.beta.kubernetes.io/region": "region1",
|
||||
"failure-domain.beta.kubernetes.io/zone": "zone2",
|
||||
"foo": "bar",
|
||||
}
|
||||
k8sNodes[2].ObjectMeta.Labels = map[string]string{}
|
||||
|
||||
retNodes = RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes)
|
||||
assert.Contains(t, retNodes[0].Location, "region=region1")
|
||||
assert.Contains(t, retNodes[0].Location, "zone=zone1")
|
||||
assert.Contains(t, retNodes[1].Location, "region=region1")
|
||||
assert.Contains(t, retNodes[1].Location, "zone=zone2")
|
||||
assert.NotContains(t, retNodes[1].Location, "foo=bar")
|
||||
assert.NotContains(t, retNodes[2].Location, "region")
|
||||
assert.NotContains(t, retNodes[2].Location, "zone")
|
||||
|
||||
}
|
||||
|
||||
func TestNodeIsInRookList(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user