diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 4fd1e90b2..9cf2542d0 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -71,7 +71,9 @@ For more details on the mons and when to choose a number other than `3`, see the - `resources`: [resources configuration settings](#cluster-wide-resources-configuration-settings) - `storage`: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes. - `useAllNodes`: `true` or `false`, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values. - If individual nodes are specified under the `nodes` field below, then `useAllNodes` must be set to `false`. + If individual nodes are specified under the `nodes` field, then `useAllNodes` must be set to `false`. + - `topologyAware`: `true` or `false`, indicating whether Rook will look for and use topology/failure domain labels on Kubernetes nodes (e.g. "region" or "zone") as part of the CRUSH location for OSDs. + Node labels must follow the formatting of the failure domain [well-known labels](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/). - `nodes`: Names of individual nodes in the cluster that should have their storage included in accordance with either the cluster level configuration specified above or any node specific overrides described in the next section below. `useAllNodes` must be set to `false` to use specific nodes and their config. See [node settings](#node-settings) below. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index e20daa5f1..dfa43518f 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -4,7 +4,9 @@ ## Notable Features -### +### Ceph + +- Rook can now be configured to read "region" and "zone" labels on Kubernetes nodes and use that information as part of the CRUSH location for the OSDs. ## Breaking Changes diff --git a/pkg/apis/rook.io/v1alpha2/types.go b/pkg/apis/rook.io/v1alpha2/types.go index b5e82ff76..1abf7a800 100644 --- a/pkg/apis/rook.io/v1alpha2/types.go +++ b/pkg/apis/rook.io/v1alpha2/types.go @@ -27,11 +27,12 @@ import ( // ************************************************************************************ type StorageScopeSpec struct { - Nodes []Node `json:"nodes,omitempty"` - UseAllNodes bool `json:"useAllNodes,omitempty"` - NodeCount int `json:"nodeCount,omitempty"` - Location string `json:"location,omitempty"` - Config map[string]string `json:"config"` + Nodes []Node `json:"nodes,omitempty"` + UseAllNodes bool `json:"useAllNodes,omitempty"` + TopologyAware bool `json:"topologyAware,omitempty"` + NodeCount int `json:"nodeCount,omitempty"` + Location string `json:"location,omitempty"` + Config map[string]string `json:"config"` Selection } diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index b0108db5c..403c2f9b8 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -157,7 +157,7 @@ func (c *Cluster) Start() error { logger.Debugf("storage nodes: %+v", c.DesiredStorage.Nodes) } // generally speaking, this finds nodes which are capable of running new osds - validNodes := k8sutil.GetValidNodes(c.DesiredStorage.Nodes, c.context.Clientset, c.placement) + validNodes := k8sutil.GetValidNodes(c.DesiredStorage, c.context.Clientset, c.placement) // no valid node is ready to run an osd if len(validNodes) == 0 { diff --git a/pkg/operator/edgefs/cluster/utils.go b/pkg/operator/edgefs/cluster/utils.go index 77ac05cb4..67820f957 100644 --- a/pkg/operator/edgefs/cluster/utils.go +++ b/pkg/operator/edgefs/cluster/utils.go @@ -93,7 +93,7 @@ func (c *cluster) getClusterNodes() ([]rookalpha.Node, error) { c.Spec.Storage.Nodes = append(c.Spec.Storage.Nodes, storageNode) } } - validNodes := k8sutil.GetValidNodes(c.Spec.Storage.Nodes, c.context.Clientset, edgefsv1beta1.GetTargetPlacement(c.Spec.Placement)) + validNodes := k8sutil.GetValidNodes(c.Spec.Storage, c.context.Clientset, edgefsv1beta1.GetTargetPlacement(c.Spec.Placement)) c.Spec.Storage.Nodes = validNodes return validNodes, nil } diff --git a/pkg/operator/k8sutil/node.go b/pkg/operator/k8sutil/node.go index 0bbd4874f..107296c81 100644 --- a/pkg/operator/k8sutil/node.go +++ b/pkg/operator/k8sutil/node.go @@ -19,6 +19,7 @@ package k8sutil import ( "fmt" + "strings" rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" v1 "k8s.io/api/core/v1" @@ -28,6 +29,11 @@ import ( helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" ) +var validTopologyLabelKeys = []string{ + "failure-domain.beta.kubernetes.io", + "failure-domain.kubernetes.io", +} + // ValidNode returns true if the node (1) is schedulable, (2) meets Rook's placement terms, and // (3) is ready. False otherwise. func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) { @@ -52,8 +58,8 @@ func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) { // GetValidNodes returns all nodes that (1) are not cordoned, (2) meet Rook's placement terms, and // (3) are ready. -func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, placement rookalpha.Placement) []rookalpha.Node { - matchingK8sNodes, err := GetKubernetesNodesMatchingRookNodes(rookNodes, clientset) +func GetValidNodes(rookStorage rookalpha.StorageScopeSpec, clientset kubernetes.Interface, placement rookalpha.Placement) []rookalpha.Node { + matchingK8sNodes, err := GetKubernetesNodesMatchingRookNodes(rookStorage.Nodes, clientset) if err != nil { // cannot list nodes, return empty nodes logger.Errorf("failed to list nodes: %+v", err) @@ -70,7 +76,7 @@ func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, p } } - return RookNodesMatchingKubernetesNodes(rookNodes, validK8sNodes) + return RookNodesMatchingKubernetesNodes(rookStorage, validK8sNodes) } // GetNodeNameFromHostname returns the name of the node resource looked up by the hostname label @@ -224,12 +230,18 @@ func GetKubernetesNodesMatchingRookNodes(rookNodes []rookalpha.Node, clientset k // RookNodesMatchingKubernetesNodes returns only the given Rook nodes which have a corresponding // match in the list of Kubernetes nodes. -func RookNodesMatchingKubernetesNodes(rookNodes []rookalpha.Node, kubernetesNodes []v1.Node) []rookalpha.Node { +func RookNodesMatchingKubernetesNodes(rookStorage rookalpha.StorageScopeSpec, kubernetesNodes []v1.Node) []rookalpha.Node { nodes := []rookalpha.Node{} for _, kn := range kubernetesNodes { - for _, rn := range rookNodes { + for _, rn := range rookStorage.Nodes { if rookNodeMatchesKubernetesNode(rn, kn) { rn.Name = normalizeHostname(kn) + + // modify the node Location if topology awareness enabled + if rookStorage.TopologyAware == true { + rn.Location = nodeTopologyLocation(kn, rn.Location) + } + nodes = append(nodes, rn) } } @@ -237,6 +249,27 @@ func RookNodesMatchingKubernetesNodes(rookNodes []rookalpha.Node, kubernetesNode return nodes } +func nodeTopologyLocation(kubeNode v1.Node, location string) string { + nodeLabels := kubeNode.ObjectMeta.Labels + locations := []string{location} + + // We're looking for node labels that match the following format: + // /: + // Where validTopologyLabelKey is an entry in the + // validTopologyLabelKeys list, key is a topology element (e.g. region, + // zone), and value is the name of the element (e.g. region1, zone2) + for label := range nodeLabels { + for _, key := range validTopologyLabelKeys { + if strings.Contains(label, key) { + keys := strings.Split(label, "/") + locations = append(locations, fmt.Sprintf("%s=%s", keys[1], nodeLabels[label])) + } + } + } + + return strings.Join(locations, " ") +} + // NodeIsInRookNodeList will return true if the target node is found in a given list of Rook nodes. func NodeIsInRookNodeList(targetNodeName string, rookNodes []rookalpha.Node) bool { for _, rn := range rookNodes { diff --git a/pkg/operator/k8sutil/node_test.go b/pkg/operator/k8sutil/node_test.go index 42d678d6c..1e116e6f1 100644 --- a/pkg/operator/k8sutil/node_test.go +++ b/pkg/operator/k8sutil/node_test.go @@ -50,12 +50,14 @@ func TestValidNode(t *testing.T) { nodeA := "nodeA" nodeB := "nodeB" - nodes := []rookalpha.Node{ - { - Name: nodeA, - }, - { - Name: nodeB, + storage := rookalpha.StorageScopeSpec{ + Nodes: []rookalpha.Node{ + { + Name: nodeA, + }, + { + Name: nodeB, + }, }, } var placement rookalpha.Placement @@ -66,7 +68,7 @@ func TestValidNode(t *testing.T) { assert.Nil(t, nodeErr) nodeErr = createNode(nodeB, v1.NodeNetworkUnavailable, clientset) assert.Nil(t, nodeErr) - validNodes := GetValidNodes(nodes, clientset, placement) + validNodes := GetValidNodes(storage, clientset, placement) assert.Equal(t, len(validNodes), 1) } @@ -265,16 +267,18 @@ func TestRookNodesMatchingKubernetesNodes(t *testing.T) { k8sNodes := []v1.Node{n0, n1, n2} // no rook nodes specified for input - rookNodes := []rookalpha.Node{} - retNodes := RookNodesMatchingKubernetesNodes(rookNodes, k8sNodes) + rookStorage := rookalpha.StorageScopeSpec{ + Nodes: []rookalpha.Node{}, + } + retNodes := RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes) assert.Len(t, retNodes, 0) // all rook nodes specified - rookNodes = []rookalpha.Node{ + rookStorage.Nodes = []rookalpha.Node{ {Name: "node0"}, {Name: "node1"}, {Name: "node2"}} - retNodes = RookNodesMatchingKubernetesNodes(rookNodes, k8sNodes) + retNodes = RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes) assert.Len(t, retNodes, 3) // this should return nodes named by hostname if that is available assert.Contains(t, retNodes, rookalpha.Node{Name: "node0-hostname"}) @@ -282,18 +286,46 @@ func TestRookNodesMatchingKubernetesNodes(t *testing.T) { assert.Contains(t, retNodes, rookalpha.Node{Name: "node2"}) // more rook nodes specified than exist - rookNodes = []rookalpha.Node{ + rookStorage.Nodes = []rookalpha.Node{ {Name: "node0-hostname"}, {Name: "node2"}, {Name: "node5"}} - retNodes = RookNodesMatchingKubernetesNodes(rookNodes, k8sNodes) + retNodes = RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes) assert.Len(t, retNodes, 2) assert.Contains(t, retNodes, rookalpha.Node{Name: "node0-hostname"}) assert.Contains(t, retNodes, rookalpha.Node{Name: "node2"}) // no k8s nodes specified - retNodes = RookNodesMatchingKubernetesNodes(rookNodes, []v1.Node{}) + retNodes = RookNodesMatchingKubernetesNodes(rookStorage, []v1.Node{}) assert.Len(t, retNodes, 0) + + // topology awareness + rookStorage.Nodes = []rookalpha.Node{ + {Name: "node0"}, + {Name: "node1"}, + {Name: "node2"}} + rookStorage.TopologyAware = true + + k8sNodes[0].ObjectMeta.Labels = map[string]string{ + "failure-domain.beta.kubernetes.io/region": "region1", + "failure-domain.beta.kubernetes.io/zone": "zone1", + } + k8sNodes[1].ObjectMeta.Labels = map[string]string{ + "failure-domain.beta.kubernetes.io/region": "region1", + "failure-domain.beta.kubernetes.io/zone": "zone2", + "foo": "bar", + } + k8sNodes[2].ObjectMeta.Labels = map[string]string{} + + retNodes = RookNodesMatchingKubernetesNodes(rookStorage, k8sNodes) + assert.Contains(t, retNodes[0].Location, "region=region1") + assert.Contains(t, retNodes[0].Location, "zone=zone1") + assert.Contains(t, retNodes[1].Location, "region=region1") + assert.Contains(t, retNodes[1].Location, "zone=zone2") + assert.NotContains(t, retNodes[1].Location, "foo=bar") + assert.NotContains(t, retNodes[2].Location, "region") + assert.NotContains(t, retNodes[2].Location, "zone") + } func TestNodeIsInRookList(t *testing.T) {