Merge pull request #1936 from rootfs/valid-node
validate storage nodes before preparing osds
This commit is contained in:
7 files changed
+216
-59
No files matched your search
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/mon"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -200,7 +201,10 @@ func (c *Cluster) checkMonsOnValidNodes() (bool, error) {
|
||||
return true, err
|
||||
}
|
||||
// check if node the mon is on is still valid
|
||||
if !validNode(*node, c.placement) {
|
||||
valid, err := k8sutil.ValidNode(*node, c.placement)
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if !valid {
|
||||
logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon)
|
||||
c.failoverMon(mon)
|
||||
return true, nil
|
||||
|
||||
@@ -444,7 +444,10 @@ func (c *Cluster) getMonNodes() ([]v1.Node, error) {
|
||||
if c.AllowMultiplePerNode && len(availableNodes) == 0 {
|
||||
logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items))
|
||||
for _, node := range nodes.Items {
|
||||
if validNode(node, c.placement) {
|
||||
valid, err := k8sutil.ValidNode(node, c.placement)
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if valid {
|
||||
availableNodes = append(availableNodes, node)
|
||||
}
|
||||
}
|
||||
@@ -472,8 +475,13 @@ func (c *Cluster) getAvailableMonNodes() ([]v1.Node, *v1.NodeList, error) {
|
||||
// choose nodes for the new mons that don't have mons currently
|
||||
availableNodes := []v1.Node{}
|
||||
for _, node := range nodes.Items {
|
||||
if !nodesInUse.Contains(node.Name) && validNode(node, c.placement) {
|
||||
availableNodes = append(availableNodes, node)
|
||||
if !nodesInUse.Contains(node.Name) {
|
||||
valid, err := k8sutil.ValidNode(node, c.placement)
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if valid {
|
||||
availableNodes = append(availableNodes, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/mon"
|
||||
@@ -37,9 +36,7 @@ import (
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
helper "k8s.io/kubernetes/pkg/api/v1/helper"
|
||||
)
|
||||
|
||||
// LoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID
|
||||
@@ -202,57 +199,6 @@ func monInQuorum(monitor client.MonMapEntry, quorum []int) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func validNode(node v1.Node, placement rookalpha.Placement) bool {
|
||||
// a node cannot be disabled
|
||||
if node.Spec.Unschedulable {
|
||||
return false
|
||||
}
|
||||
|
||||
// a node matches the NodeAffinity configuration
|
||||
// ignoring `PreferredDuringSchedulingIgnoredDuringExecution` terms: they
|
||||
// should not be used to judge a node unusable
|
||||
if placement.NodeAffinity != nil && placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
|
||||
nodeMatches := false
|
||||
for _, req := range placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
|
||||
nodeSelector, err := helper.NodeSelectorRequirementsAsSelector(req.MatchExpressions)
|
||||
if err != nil {
|
||||
logger.Infof("failed to parse MatchExpressions: %+v, regarding as not match.", req.MatchExpressions)
|
||||
return false
|
||||
}
|
||||
if nodeSelector.Matches(labels.Set(node.Labels)) {
|
||||
nodeMatches = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !nodeMatches {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// a node is tainted and cannot be tolerated
|
||||
for _, taint := range node.Spec.Taints {
|
||||
isTolerated := false
|
||||
for _, toleration := range placement.Tolerations {
|
||||
if toleration.ToleratesTaint(&taint) {
|
||||
isTolerated = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !isTolerated {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// a node must be Ready
|
||||
for _, c := range node.Status.Conditions {
|
||||
if c.Type == v1.NodeReady {
|
||||
return true
|
||||
}
|
||||
}
|
||||
logger.Infof("node %s is not ready. %+v", node.Name, node.Status.Conditions)
|
||||
return false
|
||||
}
|
||||
|
||||
// create new cluster info (FSID, shared keys)
|
||||
func createNamedClusterInfo(context *clusterd.Context, clusterName string) (*mon.ClusterInfo, error) {
|
||||
fsid, err := uuid.NewRandom()
|
||||
|
||||
@@ -159,7 +159,9 @@ func (c *Cluster) Start() error {
|
||||
}
|
||||
logger.Debugf("storage nodes: %+v", c.Storage.Nodes)
|
||||
}
|
||||
|
||||
validNodes := k8sutil.GetValidNodes(c.Storage.Nodes, c.context.Clientset, c.placement)
|
||||
logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.Storage.Nodes))
|
||||
c.Storage.Nodes = validNodes
|
||||
// orchestrate individual nodes, starting with any that are still ongoing (in the case that we
|
||||
// are resuming a previous orchestration attempt)
|
||||
config := newProvisionConfig()
|
||||
|
||||
@@ -70,6 +70,23 @@ func createDiscoverConfigmap(nodeName, ns string, clientset *fake.Clientset) err
|
||||
return err
|
||||
}
|
||||
|
||||
func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake.Clientset) error {
|
||||
node := &v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: nodeName,
|
||||
},
|
||||
Status: v1.NodeStatus{
|
||||
Conditions: []v1.NodeCondition{
|
||||
{
|
||||
Type: condition,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := clientset.CoreV1().Nodes().Create(node)
|
||||
return err
|
||||
}
|
||||
|
||||
func TestAddRemoveNode(t *testing.T) {
|
||||
// create a storage spec with the given nodes/devices/dirs
|
||||
nodeName := "node8230"
|
||||
@@ -90,6 +107,8 @@ func TestAddRemoveNode(t *testing.T) {
|
||||
os.Setenv(k8sutil.PodNamespaceEnvVar, "rook-system")
|
||||
defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
|
||||
|
||||
nodeErr := createNode(nodeName, v1.NodeReady, clientset)
|
||||
assert.Nil(t, nodeErr)
|
||||
cmErr := createDiscoverConfigmap(nodeName, "rook-system", clientset)
|
||||
assert.Nil(t, cmErr)
|
||||
|
||||
@@ -279,6 +298,8 @@ func TestAddNodeFailure(t *testing.T) {
|
||||
clientset.PrependReactor("create", "jobs", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
|
||||
return true, nil, fmt.Errorf("mock failed to create jobs")
|
||||
})
|
||||
nodeErr := createNode(nodeName, v1.NodeReady, clientset)
|
||||
assert.Nil(t, nodeErr)
|
||||
|
||||
os.Setenv(k8sutil.PodNamespaceEnvVar, "rook-system")
|
||||
defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package k8sutil for Kubernetes helpers.
|
||||
package k8sutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
helper "k8s.io/kubernetes/pkg/api/v1/helper"
|
||||
)
|
||||
|
||||
func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) {
|
||||
// a node cannot be disabled
|
||||
if node.Spec.Unschedulable {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// a node matches the NodeAffinity configuration
|
||||
// ignoring `PreferredDuringSchedulingIgnoredDuringExecution` terms: they
|
||||
// should not be used to judge a node unusable
|
||||
if placement.NodeAffinity != nil && placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
|
||||
nodeMatches := false
|
||||
for _, req := range placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
|
||||
nodeSelector, err := helper.NodeSelectorRequirementsAsSelector(req.MatchExpressions)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse MatchExpressions: %+v, regarding as not match.", req.MatchExpressions)
|
||||
}
|
||||
if nodeSelector.Matches(labels.Set(node.Labels)) {
|
||||
nodeMatches = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !nodeMatches {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// a node is tainted and cannot be tolerated
|
||||
for _, taint := range node.Spec.Taints {
|
||||
isTolerated := false
|
||||
for _, toleration := range placement.Tolerations {
|
||||
if toleration.ToleratesTaint(&taint) {
|
||||
isTolerated = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !isTolerated {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// a node must be Ready
|
||||
for _, c := range node.Status.Conditions {
|
||||
if c.Type == v1.NodeReady {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
logger.Infof("node %s is not ready. %+v", node.Name, node.Status.Conditions)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, placement rookalpha.Placement) []rookalpha.Node {
|
||||
validNodes := []rookalpha.Node{}
|
||||
|
||||
nodeOptions := metav1.ListOptions{}
|
||||
nodeOptions.TypeMeta.Kind = "Node"
|
||||
allNodes, err := clientset.CoreV1().Nodes().List(nodeOptions)
|
||||
if err != nil {
|
||||
// cannot list nodes, return empty nodes
|
||||
logger.Warningf("failed to list nodes: %v", err)
|
||||
return validNodes
|
||||
}
|
||||
|
||||
for _, node := range allNodes.Items {
|
||||
for _, rookNode := range rookNodes {
|
||||
if rookNode.Name == node.Name {
|
||||
valid, err := ValidNode(node, placement)
|
||||
if err != nil {
|
||||
logger.Warning("failed to validate node %s %v", node.Name, err)
|
||||
} else if valid {
|
||||
validNodes = append(validNodes, rookNode)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return validNodes
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package k8sutil for Kubernetes helpers.
|
||||
package k8sutil
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake.Clientset) error {
|
||||
node := &v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: nodeName,
|
||||
},
|
||||
Status: v1.NodeStatus{
|
||||
Conditions: []v1.NodeCondition{
|
||||
{
|
||||
Type: condition,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := clientset.CoreV1().Nodes().Create(node)
|
||||
return err
|
||||
}
|
||||
|
||||
func TestValidNode(t *testing.T) {
|
||||
nodeA := "nodeA"
|
||||
nodeB := "nodeB"
|
||||
|
||||
nodes := []rookalpha.Node{
|
||||
{
|
||||
Name: nodeA,
|
||||
},
|
||||
{
|
||||
Name: nodeB,
|
||||
},
|
||||
}
|
||||
var placement rookalpha.Placement
|
||||
// set up a fake k8s client set and watcher to generate events that the operator will listen to
|
||||
clientset := fake.NewSimpleClientset()
|
||||
|
||||
nodeErr := createNode(nodeA, v1.NodeReady, clientset)
|
||||
assert.Nil(t, nodeErr)
|
||||
nodeErr = createNode(nodeB, v1.NodeNetworkUnavailable, clientset)
|
||||
assert.Nil(t, nodeErr)
|
||||
validNodes := GetValidNodes(nodes, clientset, placement)
|
||||
assert.Equal(t, len(validNodes), 1)
|
||||
}
|
||||
Reference in new issue
Block a user