From 7fbe4e7db5f4845e98e444460ff5e835d0f66f3f Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Wed, 30 Oct 2024 17:00:22 -0600 Subject: [PATCH] osd: allow scheduling on unschedulable nodes For nodes that are explicitly requested for deploying OSDs, they will be skipped temporarily if not schedulable or not ready. A future reconcile is expected to schedule them. Allow the OSDs to be scheduled even on these nodes that are temporarily down, even if it blocks the reconcile from completing. Signed-off-by: Travis Nielsen --- .../CRDs/Cluster/ceph-cluster-crd.md | 2 + Documentation/CRDs/specification.md | 12 ++++++ .../charts/rook-ceph/templates/resources.yaml | 3 ++ deploy/examples/cluster.yaml | 3 ++ deploy/examples/crds.yaml | 3 ++ pkg/apis/ceph.rook.io/v1/types.go | 3 ++ pkg/operator/ceph/cluster/watcher.go | 4 +- pkg/operator/k8sutil/node.go | 26 +++++++++---- pkg/operator/k8sutil/node_test.go | 38 ++++++++++++------- 9 files changed, 70 insertions(+), 24 deletions(-) diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index fce660364..e6bba7ff5 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -94,6 +94,8 @@ For more details on the mons and when to choose a number other than `3`, see the * For non-PVCs: `placement.all` and `placement.osd` * For PVCs: `placement.all` and inside the storageClassDeviceSets from the `placement` or `preparePlacement` * `flappingRestartIntervalHours`: Defines the time for which an OSD pod will sleep before restarting, if it stopped due to flapping. Flapping occurs where OSDs are marked `down` by Ceph more than 5 times in 600 seconds. The OSDs will stay down when flapping since they likely have a bad disk or other issue that needs investigation. If the issue with the OSD is fixed manually, the OSD pod can be manually restarted. The sleep is disabled if this interval is set to 0. + * `scheduleAlways`: Whether to always schedule OSD pods on nodes declared explicitly in the "nodes" section, even if they are + temporarily not schedulable. If set to true, consider adding placement tolerations for unschedulable nodes. * `fullRatio`: The ratio at which Ceph should block IO if the OSDs are too full. The default is 0.95. * `backfillFullRatio`: The ratio at which Ceph should stop backfilling data if the OSDs are too full. The default is 0.90. * `nearFullRatio`: The ratio at which Ceph should raise a health warning if the cluster is almost full. The default is 0.85. diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index e0b015109..89f570a27 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -12821,6 +12821,18 @@ bool +scheduleAlways
+ +bool + + + +(Optional) +

Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready

+ + + + onlyApplyOSDPlacement
bool diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 77ff73089..9f0e70332 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -3645,6 +3645,9 @@ spec: type: array onlyApplyOSDPlacement: type: boolean + scheduleAlways: + description: Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready + type: boolean storageClassDeviceSets: items: description: StorageClassDeviceSet is a storage class device set diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index 345897fb5..e1ec44c23 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -285,6 +285,9 @@ spec: # config: # configuration can be specified at the node level which overrides the cluster level config # - name: "172.17.4.301" # deviceFilter: "^sd." + # Whether to always schedule OSD pods on nodes declared explicitly in the "nodes" section, even if they are + # temporarily not schedulable. If set to true, consider adding placement tolerations for unschedulable nodes. + scheduleAlways: false # when onlyApplyOSDPlacement is false, will merge both placement.All() and placement.osd onlyApplyOSDPlacement: false # Time for which an OSD pod will sleep before restarting, if it stopped due to flapping diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index b10029784..6cbfc7321 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -3643,6 +3643,9 @@ spec: type: array onlyApplyOSDPlacement: type: boolean + scheduleAlways: + description: Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready + type: boolean storageClassDeviceSets: items: description: StorageClassDeviceSet is a storage class device set diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 633b9153c..da0ef9b58 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -3035,6 +3035,9 @@ type StorageScopeSpec struct { // +optional UseAllNodes bool `json:"useAllNodes,omitempty"` // +optional + // Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready + ScheduleAlways bool `json:"scheduleAlways,omitempty"` + // +optional OnlyApplyOSDPlacement bool `json:"onlyApplyOSDPlacement,omitempty"` // +kubebuilder:pruning:PreserveUnknownFields // +nullable diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index 41cd44c1b..a99d9f545 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -100,7 +100,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, op return false } - if !k8sutil.GetNodeSchedulable(*node) { + if !k8sutil.GetNodeSchedulable(*node, false) { logger.Debugf("node watcher: skipping cluster update. added node %q is unschedulable", node.Labels[corev1.LabelHostname]) return false } @@ -123,7 +123,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, op logger.Debugf("node %q is ready, checking if it can run OSDs", node.Name) nodesCheckedForReconcile.Insert(node.Name) - err := k8sutil.ValidNode(*node, cephv1.GetOSDPlacement(cluster.Spec.Placement)) + err := k8sutil.ValidNode(*node, cephv1.GetOSDPlacement(cluster.Spec.Placement), cluster.Spec.Storage.ScheduleAlways) if err == nil { nodeName := node.Name hostname, ok := node.Labels[corev1.LabelHostname] diff --git a/pkg/operator/k8sutil/node.go b/pkg/operator/k8sutil/node.go index 0962c2692..2dcce962c 100644 --- a/pkg/operator/k8sutil/node.go +++ b/pkg/operator/k8sutil/node.go @@ -22,9 +22,10 @@ import ( "encoding/json" "errors" "fmt" - "sigs.k8s.io/yaml" "strings" + "sigs.k8s.io/yaml" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,7 +38,7 @@ import ( // validNodeNoSched returns true if the node (1) meets Rook's placement terms, // and (2) is ready. Unlike ValidNode, this method will ignore the // Node.Spec.Unschedulable flag. False otherwise. -func validNodeNoSched(node v1.Node, placement cephv1.Placement) error { +func validNodeNoSched(node v1.Node, placement cephv1.Placement, scheduleAlways bool) error { p, err := NodeMeetsPlacementTerms(node, placement, false) if err != nil { return fmt.Errorf("failed to check if node meets Rook placement terms. %+v", err) @@ -47,6 +48,10 @@ func validNodeNoSched(node v1.Node, placement cephv1.Placement) error { } if !NodeIsReady(node) { + if scheduleAlways { + logger.Infof("node %q is not ready but scheduleAlways set", node.Name) + return nil + } return errors.New("node is not ready") } @@ -55,12 +60,12 @@ func validNodeNoSched(node v1.Node, placement cephv1.Placement) error { // ValidNode returns true if the node (1) is schedulable, (2) meets Rook's placement terms, and // (3) is ready. False otherwise. -func ValidNode(node v1.Node, placement cephv1.Placement) error { - if !GetNodeSchedulable(node) { +func ValidNode(node v1.Node, placement cephv1.Placement, scheduleAlways bool) error { + if !GetNodeSchedulable(node, scheduleAlways) { return errors.New("node is unschedulable") } - return validNodeNoSched(node, placement) + return validNodeNoSched(node, placement, scheduleAlways) } // GetValidNodes returns all nodes that (1) are not cordoned, (2) meet Rook's placement terms, and @@ -76,7 +81,7 @@ func GetValidNodes(ctx context.Context, rookStorage cephv1.StorageScopeSpec, cli validK8sNodes := []v1.Node{} reasonsForSkippingNodes := map[string][]string{} for _, n := range matchingK8sNodes { - err := ValidNode(n, placement) + err := ValidNode(n, placement, rookStorage.ScheduleAlways) if err != nil { reason := err.Error() reasonsForSkippingNodes[reason] = append(reasonsForSkippingNodes[reason], n.Name) @@ -144,10 +149,15 @@ func GetNodeHostNames(ctx context.Context, clientset kubernetes.Interface) (map[ // GetNodeSchedulable returns a boolean if the node is tainted as Schedulable or not // true -> Node is schedulable // false -> Node is unschedulable -func GetNodeSchedulable(node v1.Node) bool { +func GetNodeSchedulable(node v1.Node, scheduleAlways bool) bool { // some unit tests set this to quickly emulate an unschedulable node; if this is set to true, // we can shortcut deeper inspection for schedulability. - return !node.Spec.Unschedulable + schedulable := !node.Spec.Unschedulable + if !schedulable && scheduleAlways { + logger.Infof("node %q is unschedulable but scheduling since scheduleAlways is set", node.Name) + return true + } + return schedulable } // NodeMeetsPlacementTerms returns true if the Rook placement allows the node to have resources scheduled diff --git a/pkg/operator/k8sutil/node_test.go b/pkg/operator/k8sutil/node_test.go index b92b3dd8c..c3cff521d 100644 --- a/pkg/operator/k8sutil/node_test.go +++ b/pkg/operator/k8sutil/node_test.go @@ -25,6 +25,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" optest "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -74,10 +75,19 @@ func TestValidNode(t *testing.T) { t.Run("test valid node", func(t *testing.T) { var placement cephv1.Placement validNodes := GetValidNodes(context.TODO(), storage, clientset, placement) - assert.Equal(t, len(validNodes), 1) + assert.Equal(t, 1, len(validNodes)) assert.Equal(t, "nodeA", validNodes[0].Name) }) + t.Run("test nodes always valid", func(t *testing.T) { + var placement cephv1.Placement + storage.ScheduleAlways = true + validNodes := GetValidNodes(context.TODO(), storage, clientset, placement) + require.Equal(t, 2, len(validNodes)) + assert.Equal(t, "nodeA", validNodes[0].Name) + assert.Equal(t, "nodeB", validNodes[1].Name) + }) + t.Run("test placement", func(t *testing.T) { nodeErr = createNode("nodeC", v1.NodeReady, clientset) assert.Nil(t, nodeErr) @@ -438,12 +448,12 @@ func TestGenerateNodeAffinity(t *testing.T) { name: "GenerateNodeAffinityWithYAMLInputUsingDoesNotExistOperator", args: args{ nodeAffinity: ` ---- -requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - - matchExpressions: - - +--- +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - + matchExpressions: + - key: myKey operator: DoesNotExist`, }, @@ -467,15 +477,15 @@ requiredDuringSchedulingIgnoredDuringExecution: name: "GenerateNodeAffinityWithYAMLInputUsingNotInOperator", args: args{ nodeAffinity: ` ---- -requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - - matchExpressions: - - +--- +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - + matchExpressions: + - key: myKey operator: NotIn - values: + values: - myValue`, }, want: &v1.NodeAffinity{