+
onlyApplyOSDPlacement
bool
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index bd2989590..87d47294a 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -3613,6 +3613,9 @@ spec:
type: array
onlyApplyOSDPlacement:
type: boolean
+ scheduleAlways:
+ description: Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready
+ type: boolean
storageClassDeviceSets:
items:
description: StorageClassDeviceSet is a storage class device set
diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml
index 4f2515828..de194ff15 100644
--- a/deploy/examples/cluster.yaml
+++ b/deploy/examples/cluster.yaml
@@ -286,6 +286,9 @@ spec:
# config: # configuration can be specified at the node level which overrides the cluster level config
# - name: "172.17.4.301"
# deviceFilter: "^sd."
+ # Whether to always schedule OSD pods on nodes declared explicitly in the "nodes" section, even if they are
+ # temporarily not schedulable. If set to true, consider adding placement tolerations for unschedulable nodes.
+ scheduleAlways: false
# when onlyApplyOSDPlacement is false, will merge both placement.All() and placement.osd
onlyApplyOSDPlacement: false
# Time for which an OSD pod will sleep before restarting, if it stopped due to flapping
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 0d57c5108..71ff3f839 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -3611,6 +3611,9 @@ spec:
type: array
onlyApplyOSDPlacement:
type: boolean
+ scheduleAlways:
+ description: Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready
+ type: boolean
storageClassDeviceSets:
items:
description: StorageClassDeviceSet is a storage class device set
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 170fb237f..a7cfdaa64 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -3033,6 +3033,9 @@ type StorageScopeSpec struct {
// +optional
UseAllNodes bool `json:"useAllNodes,omitempty"`
// +optional
+ // Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready
+ ScheduleAlways bool `json:"scheduleAlways,omitempty"`
+ // +optional
OnlyApplyOSDPlacement bool `json:"onlyApplyOSDPlacement,omitempty"`
// +kubebuilder:pruning:PreserveUnknownFields
// +nullable
diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go
index 41cd44c1b..a99d9f545 100644
--- a/pkg/operator/ceph/cluster/watcher.go
+++ b/pkg/operator/ceph/cluster/watcher.go
@@ -100,7 +100,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, op
return false
}
- if !k8sutil.GetNodeSchedulable(*node) {
+ if !k8sutil.GetNodeSchedulable(*node, false) {
logger.Debugf("node watcher: skipping cluster update. added node %q is unschedulable", node.Labels[corev1.LabelHostname])
return false
}
@@ -123,7 +123,7 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, op
logger.Debugf("node %q is ready, checking if it can run OSDs", node.Name)
nodesCheckedForReconcile.Insert(node.Name)
- err := k8sutil.ValidNode(*node, cephv1.GetOSDPlacement(cluster.Spec.Placement))
+ err := k8sutil.ValidNode(*node, cephv1.GetOSDPlacement(cluster.Spec.Placement), cluster.Spec.Storage.ScheduleAlways)
if err == nil {
nodeName := node.Name
hostname, ok := node.Labels[corev1.LabelHostname]
diff --git a/pkg/operator/k8sutil/node.go b/pkg/operator/k8sutil/node.go
index 0962c2692..2dcce962c 100644
--- a/pkg/operator/k8sutil/node.go
+++ b/pkg/operator/k8sutil/node.go
@@ -22,9 +22,10 @@ import (
"encoding/json"
"errors"
"fmt"
- "sigs.k8s.io/yaml"
"strings"
+ "sigs.k8s.io/yaml"
+
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -37,7 +38,7 @@ import (
// validNodeNoSched returns true if the node (1) meets Rook's placement terms,
// and (2) is ready. Unlike ValidNode, this method will ignore the
// Node.Spec.Unschedulable flag. False otherwise.
-func validNodeNoSched(node v1.Node, placement cephv1.Placement) error {
+func validNodeNoSched(node v1.Node, placement cephv1.Placement, scheduleAlways bool) error {
p, err := NodeMeetsPlacementTerms(node, placement, false)
if err != nil {
return fmt.Errorf("failed to check if node meets Rook placement terms. %+v", err)
@@ -47,6 +48,10 @@ func validNodeNoSched(node v1.Node, placement cephv1.Placement) error {
}
if !NodeIsReady(node) {
+ if scheduleAlways {
+ logger.Infof("node %q is not ready but scheduleAlways set", node.Name)
+ return nil
+ }
return errors.New("node is not ready")
}
@@ -55,12 +60,12 @@ func validNodeNoSched(node v1.Node, placement cephv1.Placement) error {
// ValidNode returns true if the node (1) is schedulable, (2) meets Rook's placement terms, and
// (3) is ready. False otherwise.
-func ValidNode(node v1.Node, placement cephv1.Placement) error {
- if !GetNodeSchedulable(node) {
+func ValidNode(node v1.Node, placement cephv1.Placement, scheduleAlways bool) error {
+ if !GetNodeSchedulable(node, scheduleAlways) {
return errors.New("node is unschedulable")
}
- return validNodeNoSched(node, placement)
+ return validNodeNoSched(node, placement, scheduleAlways)
}
// GetValidNodes returns all nodes that (1) are not cordoned, (2) meet Rook's placement terms, and
@@ -76,7 +81,7 @@ func GetValidNodes(ctx context.Context, rookStorage cephv1.StorageScopeSpec, cli
validK8sNodes := []v1.Node{}
reasonsForSkippingNodes := map[string][]string{}
for _, n := range matchingK8sNodes {
- err := ValidNode(n, placement)
+ err := ValidNode(n, placement, rookStorage.ScheduleAlways)
if err != nil {
reason := err.Error()
reasonsForSkippingNodes[reason] = append(reasonsForSkippingNodes[reason], n.Name)
@@ -144,10 +149,15 @@ func GetNodeHostNames(ctx context.Context, clientset kubernetes.Interface) (map[
// GetNodeSchedulable returns a boolean if the node is tainted as Schedulable or not
// true -> Node is schedulable
// false -> Node is unschedulable
-func GetNodeSchedulable(node v1.Node) bool {
+func GetNodeSchedulable(node v1.Node, scheduleAlways bool) bool {
// some unit tests set this to quickly emulate an unschedulable node; if this is set to true,
// we can shortcut deeper inspection for schedulability.
- return !node.Spec.Unschedulable
+ schedulable := !node.Spec.Unschedulable
+ if !schedulable && scheduleAlways {
+ logger.Infof("node %q is unschedulable but scheduling since scheduleAlways is set", node.Name)
+ return true
+ }
+ return schedulable
}
// NodeMeetsPlacementTerms returns true if the Rook placement allows the node to have resources scheduled
diff --git a/pkg/operator/k8sutil/node_test.go b/pkg/operator/k8sutil/node_test.go
index b92b3dd8c..c3cff521d 100644
--- a/pkg/operator/k8sutil/node_test.go
+++ b/pkg/operator/k8sutil/node_test.go
@@ -25,6 +25,7 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
optest "github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
@@ -74,10 +75,19 @@ func TestValidNode(t *testing.T) {
t.Run("test valid node", func(t *testing.T) {
var placement cephv1.Placement
validNodes := GetValidNodes(context.TODO(), storage, clientset, placement)
- assert.Equal(t, len(validNodes), 1)
+ assert.Equal(t, 1, len(validNodes))
assert.Equal(t, "nodeA", validNodes[0].Name)
})
+ t.Run("test nodes always valid", func(t *testing.T) {
+ var placement cephv1.Placement
+ storage.ScheduleAlways = true
+ validNodes := GetValidNodes(context.TODO(), storage, clientset, placement)
+ require.Equal(t, 2, len(validNodes))
+ assert.Equal(t, "nodeA", validNodes[0].Name)
+ assert.Equal(t, "nodeB", validNodes[1].Name)
+ })
+
t.Run("test placement", func(t *testing.T) {
nodeErr = createNode("nodeC", v1.NodeReady, clientset)
assert.Nil(t, nodeErr)
@@ -438,12 +448,12 @@ func TestGenerateNodeAffinity(t *testing.T) {
name: "GenerateNodeAffinityWithYAMLInputUsingDoesNotExistOperator",
args: args{
nodeAffinity: `
----
-requiredDuringSchedulingIgnoredDuringExecution:
- nodeSelectorTerms:
- -
- matchExpressions:
- -
+---
+requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ -
+ matchExpressions:
+ -
key: myKey
operator: DoesNotExist`,
},
@@ -467,15 +477,15 @@ requiredDuringSchedulingIgnoredDuringExecution:
name: "GenerateNodeAffinityWithYAMLInputUsingNotInOperator",
args: args{
nodeAffinity: `
----
-requiredDuringSchedulingIgnoredDuringExecution:
- nodeSelectorTerms:
- -
- matchExpressions:
- -
+---
+requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ -
+ matchExpressions:
+ -
key: myKey
operator: NotIn
- values:
+ values:
- myValue`,
},
want: &v1.NodeAffinity{
|