From c6f463d0e7a514618585fdf23c13370d1a43217c Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Tue, 24 Jul 2018 19:13:56 -0400 Subject: [PATCH] validate storage nodes before preparing osds Signed-off-by: Huamin Chen --- pkg/operator/ceph/cluster/mon/health.go | 6 +- pkg/operator/ceph/cluster/mon/mon.go | 14 ++- pkg/operator/ceph/cluster/mon/util.go | 54 ----------- pkg/operator/ceph/cluster/osd/osd.go | 4 +- pkg/operator/ceph/cluster/osd/osd_test.go | 21 +++++ pkg/operator/k8sutil/node.go | 107 ++++++++++++++++++++++ pkg/operator/k8sutil/node_test.go | 69 ++++++++++++++ 7 files changed, 216 insertions(+), 59 deletions(-) create mode 100644 pkg/operator/k8sutil/node.go create mode 100644 pkg/operator/k8sutil/node_test.go diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go index 51eed21c4..6e034b3be 100644 --- a/pkg/operator/ceph/cluster/mon/health.go +++ b/pkg/operator/ceph/cluster/mon/health.go @@ -24,6 +24,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/daemon/ceph/mon" + "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -200,7 +201,10 @@ func (c *Cluster) checkMonsOnValidNodes() (bool, error) { return true, err } // check if node the mon is on is still valid - if !validNode(*node, c.placement) { + valid, err := k8sutil.ValidNode(*node, c.placement) + if err != nil { + logger.Warning("failed to validate node %s %v", node.Name, err) + } else if !valid { logger.Warningf("node %s isn't valid anymore, failover mon %s", nInfo.Name, mon) c.failoverMon(mon) return true, nil diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 392aa2b82..3a3177961 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -444,7 +444,10 @@ func (c *Cluster) getMonNodes() ([]v1.Node, error) { if c.AllowMultiplePerNode && len(availableNodes) == 0 { logger.Infof("All nodes are running mons. Adding all %d nodes to the availability.", len(nodes.Items)) for _, node := range nodes.Items { - if validNode(node, c.placement) { + valid, err := k8sutil.ValidNode(node, c.placement) + if err != nil { + logger.Warning("failed to validate node %s %v", node.Name, err) + } else if valid { availableNodes = append(availableNodes, node) } } @@ -472,8 +475,13 @@ func (c *Cluster) getAvailableMonNodes() ([]v1.Node, *v1.NodeList, error) { // choose nodes for the new mons that don't have mons currently availableNodes := []v1.Node{} for _, node := range nodes.Items { - if !nodesInUse.Contains(node.Name) && validNode(node, c.placement) { - availableNodes = append(availableNodes, node) + if !nodesInUse.Contains(node.Name) { + valid, err := k8sutil.ValidNode(node, c.placement) + if err != nil { + logger.Warning("failed to validate node %s %v", node.Name, err) + } else if valid { + availableNodes = append(availableNodes, node) + } } } diff --git a/pkg/operator/ceph/cluster/mon/util.go b/pkg/operator/ceph/cluster/mon/util.go index f9c28dad8..02c4964fb 100644 --- a/pkg/operator/ceph/cluster/mon/util.go +++ b/pkg/operator/ceph/cluster/mon/util.go @@ -27,7 +27,6 @@ import ( "strings" "github.com/google/uuid" - rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/daemon/ceph/mon" @@ -37,9 +36,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" - helper "k8s.io/kubernetes/pkg/api/v1/helper" ) // LoadClusterInfo constructs or loads a clusterinfo and returns it along with the maxMonID @@ -202,57 +199,6 @@ func monInQuorum(monitor client.MonMapEntry, quorum []int) bool { return false } -func validNode(node v1.Node, placement rookalpha.Placement) bool { - // a node cannot be disabled - if node.Spec.Unschedulable { - return false - } - - // a node matches the NodeAffinity configuration - // ignoring `PreferredDuringSchedulingIgnoredDuringExecution` terms: they - // should not be used to judge a node unusable - if placement.NodeAffinity != nil && placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { - nodeMatches := false - for _, req := range placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { - nodeSelector, err := helper.NodeSelectorRequirementsAsSelector(req.MatchExpressions) - if err != nil { - logger.Infof("failed to parse MatchExpressions: %+v, regarding as not match.", req.MatchExpressions) - return false - } - if nodeSelector.Matches(labels.Set(node.Labels)) { - nodeMatches = true - break - } - } - if !nodeMatches { - return false - } - } - - // a node is tainted and cannot be tolerated - for _, taint := range node.Spec.Taints { - isTolerated := false - for _, toleration := range placement.Tolerations { - if toleration.ToleratesTaint(&taint) { - isTolerated = true - break - } - } - if !isTolerated { - return false - } - } - - // a node must be Ready - for _, c := range node.Status.Conditions { - if c.Type == v1.NodeReady { - return true - } - } - logger.Infof("node %s is not ready. %+v", node.Name, node.Status.Conditions) - return false -} - // create new cluster info (FSID, shared keys) func createNamedClusterInfo(context *clusterd.Context, clusterName string) (*mon.ClusterInfo, error) { fsid, err := uuid.NewRandom() diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 97f350f64..1521dbe17 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -159,7 +159,9 @@ func (c *Cluster) Start() error { } logger.Debugf("storage nodes: %+v", c.Storage.Nodes) } - + validNodes := k8sutil.GetValidNodes(c.Storage.Nodes, c.context.Clientset, c.placement) + logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.Storage.Nodes)) + c.Storage.Nodes = validNodes // orchestrate individual nodes, starting with any that are still ongoing (in the case that we // are resuming a previous orchestration attempt) config := newProvisionConfig() diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index 096f73adb..1827cc941 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -70,6 +70,23 @@ func createDiscoverConfigmap(nodeName, ns string, clientset *fake.Clientset) err return err } +func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake.Clientset) error { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + { + Type: condition, + }, + }, + }, + } + _, err := clientset.CoreV1().Nodes().Create(node) + return err +} + func TestAddRemoveNode(t *testing.T) { // create a storage spec with the given nodes/devices/dirs nodeName := "node8230" @@ -90,6 +107,8 @@ func TestAddRemoveNode(t *testing.T) { os.Setenv(k8sutil.PodNamespaceEnvVar, "rook-system") defer os.Unsetenv(k8sutil.PodNamespaceEnvVar) + nodeErr := createNode(nodeName, v1.NodeReady, clientset) + assert.Nil(t, nodeErr) cmErr := createDiscoverConfigmap(nodeName, "rook-system", clientset) assert.Nil(t, cmErr) @@ -279,6 +298,8 @@ func TestAddNodeFailure(t *testing.T) { clientset.PrependReactor("create", "jobs", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("mock failed to create jobs") }) + nodeErr := createNode(nodeName, v1.NodeReady, clientset) + assert.Nil(t, nodeErr) os.Setenv(k8sutil.PodNamespaceEnvVar, "rook-system") defer os.Unsetenv(k8sutil.PodNamespaceEnvVar) diff --git a/pkg/operator/k8sutil/node.go b/pkg/operator/k8sutil/node.go new file mode 100644 index 000000000..9d199934d --- /dev/null +++ b/pkg/operator/k8sutil/node.go @@ -0,0 +1,107 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package k8sutil for Kubernetes helpers. +package k8sutil + +import ( + "fmt" + + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + helper "k8s.io/kubernetes/pkg/api/v1/helper" +) + +func ValidNode(node v1.Node, placement rookalpha.Placement) (bool, error) { + // a node cannot be disabled + if node.Spec.Unschedulable { + return false, nil + } + + // a node matches the NodeAffinity configuration + // ignoring `PreferredDuringSchedulingIgnoredDuringExecution` terms: they + // should not be used to judge a node unusable + if placement.NodeAffinity != nil && placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + nodeMatches := false + for _, req := range placement.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + nodeSelector, err := helper.NodeSelectorRequirementsAsSelector(req.MatchExpressions) + if err != nil { + return false, fmt.Errorf("failed to parse MatchExpressions: %+v, regarding as not match.", req.MatchExpressions) + } + if nodeSelector.Matches(labels.Set(node.Labels)) { + nodeMatches = true + break + } + } + if !nodeMatches { + return false, nil + } + } + + // a node is tainted and cannot be tolerated + for _, taint := range node.Spec.Taints { + isTolerated := false + for _, toleration := range placement.Tolerations { + if toleration.ToleratesTaint(&taint) { + isTolerated = true + break + } + } + if !isTolerated { + return false, nil + } + } + + // a node must be Ready + for _, c := range node.Status.Conditions { + if c.Type == v1.NodeReady { + return true, nil + } + } + logger.Infof("node %s is not ready. %+v", node.Name, node.Status.Conditions) + return false, nil +} + +func GetValidNodes(rookNodes []rookalpha.Node, clientset kubernetes.Interface, placement rookalpha.Placement) []rookalpha.Node { + validNodes := []rookalpha.Node{} + + nodeOptions := metav1.ListOptions{} + nodeOptions.TypeMeta.Kind = "Node" + allNodes, err := clientset.CoreV1().Nodes().List(nodeOptions) + if err != nil { + // cannot list nodes, return empty nodes + logger.Warningf("failed to list nodes: %v", err) + return validNodes + } + + for _, node := range allNodes.Items { + for _, rookNode := range rookNodes { + if rookNode.Name == node.Name { + valid, err := ValidNode(node, placement) + if err != nil { + logger.Warning("failed to validate node %s %v", node.Name, err) + } else if valid { + validNodes = append(validNodes, rookNode) + } + break + } + } + } + return validNodes +} diff --git a/pkg/operator/k8sutil/node_test.go b/pkg/operator/k8sutil/node_test.go new file mode 100644 index 000000000..38f63f62c --- /dev/null +++ b/pkg/operator/k8sutil/node_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package k8sutil for Kubernetes helpers. +package k8sutil + +import ( + "testing" + + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func createNode(nodeName string, condition v1.NodeConditionType, clientset *fake.Clientset) error { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + { + Type: condition, + }, + }, + }, + } + _, err := clientset.CoreV1().Nodes().Create(node) + return err +} + +func TestValidNode(t *testing.T) { + nodeA := "nodeA" + nodeB := "nodeB" + + nodes := []rookalpha.Node{ + { + Name: nodeA, + }, + { + Name: nodeB, + }, + } + var placement rookalpha.Placement + // set up a fake k8s client set and watcher to generate events that the operator will listen to + clientset := fake.NewSimpleClientset() + + nodeErr := createNode(nodeA, v1.NodeReady, clientset) + assert.Nil(t, nodeErr) + nodeErr = createNode(nodeB, v1.NodeNetworkUnavailable, clientset) + assert.Nil(t, nodeErr) + validNodes := GetValidNodes(nodes, clientset, placement) + assert.Equal(t, len(validNodes), 1) +}