Merge pull request #1936 from rootfs/valid-node

validate storage nodes before preparing osds
This commit is contained in:
Travis Nielsen authored and GitHub committed 2018-07-26 13:03:14 -06:00
commit cee683fd4a
7 files changed
+216 -59

No files matched your search

+5 -1
View File
@@ -24,6 +24,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/daemon/ceph/mon"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -200,7 +201,10 @@ func (c *Cluster) checkMonsOnValidNodes() (bool, error) {
return true, err
}
// check if node the mon is on is still valid
if !validNode(*node, c.placement) {
valid, err := k8sutil.ValidNode(*node, c.placement)
if err != nil {
logger.Warning("failed to validate node %s %v", node.Name, err)
} else if !valid {
logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon)
c.failoverMon(mon)
return true, nil
+11 -3
View File
@@ -444,7 +444,10 @@ func (c *Cluster) getMonNodes() ([]v1.Node, error) {
if c.AllowMultiplePerNode && len(availableNodes) == 0 {
logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items))
for _, node := range nodes.Items {
if validNode(node, c.placement) {
valid, err := k8sutil.ValidNode(node, c.placement)
if err != nil {
logger.Warning("failed to validate node %s %v", node.Name, err)
} else if valid {
availableNodes = append(availableNodes, node)
}
}
@@ -472,8 +475,13 @@ func (c *Cluster) getAvailableMonNodes() ([]v1.Node, *v1.NodeList, error) {
// choose nodes for the new mons that don't have mons currently
availableNodes := []v1.Node{}
for _, node := range nodes.Items {
if !nodesInUse.Contains(node.Name) && validNode(node, c.placement) {
availableNodes = append(availableNodes, node)
if !nodesInUse.Contains(node.Name) {
valid, err := k8sutil.ValidNode(node, c.placement)
if err != nil {
logger.Warning("failed to validate node %s %v", node.Name, err)
} else if valid {
availableNodes = append(availableNodes, node)
}
}
}
-54
View File
@@ -27,7 +27,6 @@ import (
"strings"
"github.com/google/uuid"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/daemon/ceph/mon"
@@ -37,9 +36,7 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
helper "k8s.io/kubernetes/pkg/api/v1/helper"
)
// LoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID
@@ -202,57 +199,6 @@ func monInQuorum(monitor client.MonMapEntry, quorum []int) bool {
return false
}
func validNode(node v1.Node, placement rookalpha.Placement) bool {
// a node cannot be disabled
if node.Spec.Unschedulable {
return false
}
// a node matches the NodeAffinity configuration
// ignoring `PreferredDuringSchedulingIgnoredDuringExecution` terms: they
// should not be used to judge a node unusable
if placement.NodeAffinity != nil && placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
nodeMatches := false
for _, req := range placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
nodeSelector, err := helper.NodeSelectorRequirementsAsSelector(req.MatchExpressions)
if err != nil {
logger.Infof("failed to parse MatchExpressions: %+v, regarding as not match.", req.MatchExpressions)
return false
}
if nodeSelector.Matches(labels.Set(node.Labels)) {
nodeMatches = true
break
}
}
if !nodeMatches {
return false
}
}
// a node is tainted and cannot be tolerated
for _, taint := range node.Spec.Taints {
isTolerated := false
for _, toleration := range placement.Tolerations {
if toleration.ToleratesTaint(&taint) {
isTolerated = true
break
}
}
if !isTolerated {
return false
}
}
// a node must be Ready
for _, c := range node.Status.Conditions {
if c.Type == v1.NodeReady {
return true
}
}
logger.Infof("node %s is not ready. %+v", node.Name, node.Status.Conditions)
return false
}
// create new cluster info (FSID, shared keys)
func createNamedClusterInfo(context *clusterd.Context, clusterName string) (*mon.ClusterInfo, error) {
fsid, err := uuid.NewRandom()
+3 -1
View File
@@ -159,7 +159,9 @@ func (c *Cluster) Start() error {
}
logger.Debugf("storage nodes: %+v", c.Storage.Nodes)
}
validNodes := k8sutil.GetValidNodes(c.Storage.Nodes, c.context.Clientset, c.placement)
logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.Storage.Nodes))
c.Storage.Nodes = validNodes
// orchestrate individual nodes, starting with any that are still ongoing (in the case that we
// are resuming a previous orchestration attempt)
config := newProvisionConfig()
+21
View File
@@ -70,6 +70,23 @@ func createDiscoverConfigmap(nodeName, ns string, clientset *fake.Clientset) err
return err
}
func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake.Clientset) error {
node := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
},
Status: v1.NodeStatus{
Conditions: []v1.NodeCondition{
{
Type: condition,
},
},
},
}
_, err := clientset.CoreV1().Nodes().Create(node)
return err
}
func TestAddRemoveNode(t *testing.T) {
// create a storage spec with the given nodes/devices/dirs
nodeName := "node8230"
@@ -90,6 +107,8 @@ func TestAddRemoveNode(t *testing.T) {
os.Setenv(k8sutil.PodNamespaceEnvVar, "rook-system")
defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
nodeErr := createNode(nodeName, v1.NodeReady, clientset)
assert.Nil(t, nodeErr)
cmErr := createDiscoverConfigmap(nodeName, "rook-system", clientset)
assert.Nil(t, cmErr)
@@ -279,6 +298,8 @@ func TestAddNodeFailure(t *testing.T) {
clientset.PrependReactor("create", "jobs", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
return true, nil, fmt.Errorf("mock failed to create jobs")
})
nodeErr := createNode(nodeName, v1.NodeReady, clientset)
assert.Nil(t, nodeErr)
os.Setenv(k8sutil.PodNamespaceEnvVar, "rook-system")
defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
+107
View File
@@ -0,0 +1,107 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package k8sutil for Kubernetes helpers.
package k8sutil
import (
"fmt"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
helper "k8s.io/kubernetes/pkg/api/v1/helper"
)
func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) {
// a node cannot be disabled
if node.Spec.Unschedulable {
return false, nil
}
// a node matches the NodeAffinity configuration
// ignoring `PreferredDuringSchedulingIgnoredDuringExecution` terms: they
// should not be used to judge a node unusable
if placement.NodeAffinity != nil && placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
nodeMatches := false
for _, req := range placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
nodeSelector, err := helper.NodeSelectorRequirementsAsSelector(req.MatchExpressions)
if err != nil {
return false, fmt.Errorf("failed to parse MatchExpressions: %+v, regarding as not match.", req.MatchExpressions)
}
if nodeSelector.Matches(labels.Set(node.Labels)) {
nodeMatches = true
break
}
}
if !nodeMatches {
return false, nil
}
}
// a node is tainted and cannot be tolerated
for _, taint := range node.Spec.Taints {
isTolerated := false
for _, toleration := range placement.Tolerations {
if toleration.ToleratesTaint(&taint) {
isTolerated = true
break
}
}
if !isTolerated {
return false, nil
}
}
// a node must be Ready
for _, c := range node.Status.Conditions {
if c.Type == v1.NodeReady {
return true, nil
}
}
logger.Infof("node %s is not ready. %+v", node.Name, node.Status.Conditions)
return false, nil
}
func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, placement rookalpha.Placement) []rookalpha.Node {
validNodes := []rookalpha.Node{}
nodeOptions := metav1.ListOptions{}
nodeOptions.TypeMeta.Kind = "Node"
allNodes, err := clientset.CoreV1().Nodes().List(nodeOptions)
if err != nil {
// cannot list nodes, return empty nodes
logger.Warningf("failed to list nodes: %v", err)
return validNodes
}
for _, node := range allNodes.Items {
for _, rookNode := range rookNodes {
if rookNode.Name == node.Name {
valid, err := ValidNode(node, placement)
if err != nil {
logger.Warning("failed to validate node %s %v", node.Name, err)
} else if valid {
validNodes = append(validNodes, rookNode)
}
break
}
}
}
return validNodes
}
+69
View File
@@ -0,0 +1,69 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package k8sutil for Kubernetes helpers.
package k8sutil
import (
"testing"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake.Clientset) error {
node := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
},
Status: v1.NodeStatus{
Conditions: []v1.NodeCondition{
{
Type: condition,
},
},
},
}
_, err := clientset.CoreV1().Nodes().Create(node)
return err
}
func TestValidNode(t *testing.T) {
nodeA := "nodeA"
nodeB := "nodeB"
nodes := []rookalpha.Node{
{
Name: nodeA,
},
{
Name: nodeB,
},
}
var placement rookalpha.Placement
// set up a fake k8s client set and watcher to generate events that the operator will listen to
clientset := fake.NewSimpleClientset()
nodeErr := createNode(nodeA, v1.NodeReady, clientset)
assert.Nil(t, nodeErr)
nodeErr = createNode(nodeB, v1.NodeNetworkUnavailable, clientset)
assert.Nil(t, nodeErr)
validNodes := GetValidNodes(nodes, clientset, placement)
assert.Equal(t, len(validNodes), 1)
}