From 43f26534edc11f0bb74d4804d0da2a384f66d95f Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Tue, 28 Oct 2025 16:58:15 -0700 Subject: [PATCH] osd: add CephCluster.spec.storage.OSDMaxUpdatesInParallel Configures the maximum number of OSDs to update in parallel. Signed-off-by: Joshua Hoblitt --- Documentation/CRDs/Cluster/ceph-cluster-crd.md | 1 + Documentation/CRDs/specification.md | 12 ++++++++++++ deploy/charts/rook-ceph/templates/resources.yaml | 5 +++++ deploy/examples/crds.yaml | 5 +++++ pkg/apis/ceph.rook.io/v1/types.go | 4 ++++ pkg/daemon/ceph/client/osd.go | 2 +- pkg/operator/ceph/cluster/osd/update.go | 16 ++++++++++------ 7 files changed, 38 insertions(+), 7 deletions(-) diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index 4d0bbb6da..5dcf9b63a 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -88,6 +88,7 @@ For more details on the mons and when to choose a number other than `3`, see the * `allowOsdCrushWeightUpdate`: Whether Rook will resize the OSD CRUSH weight when the OSD PVC size is increased. This allows cluster data to be rebalanced to make most effective use of new OSD space. The default is false since data rebalancing can cause temporary cluster slowdown. + * `osdMaxUpdatesInParallel`: The maximum number of OSDs that are allowed to be simultaneously down during an OSD update. Note that an "update" always takes place upon operator restart and only OSDs which are `ok-to-stop` are taken down. The default value is `20`. Decreasing this value will potentially reduce the impact of updates on the cluster by keeping more OSDs online during an update. Increasing the value may reduce the total time for an update to complete. This is an advanced tuning parameter and the default value should be suitable for most clusters. * [storage selection settings](#storage-selection-settings) * [Storage Class Device Sets](#storage-class-device-sets) * `onlyApplyOSDPlacement`: Whether the placement specific for OSDs is merged with the `all` placement. If `false`, the OSD placement will be merged with the `all` placement. If true, the `OSD placement will be applied` and the `all` placement will be ignored. The placement for OSDs is computed from several different places depending on the type of OSD: diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 1e3285a8c..062227a06 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -14624,6 +14624,18 @@ This allows cluster data to be rebalanced to make most effective use of new OSD The default is false since data rebalancing can cause temporary cluster slowdown.

+ + +osdMaxUpdatesInParallel
+ +uint32 + + + +(Optional) +

The maximum number of OSDs to update in parallel.

+ +

StoreType diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index a0202ff8e..b811b0eef 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -4284,6 +4284,11 @@ spec: type: array onlyApplyOSDPlacement: type: boolean + osdMaxUpdatesInParallel: + description: The maximum number of OSDs to update in parallel. + format: int32 + minimum: 1 + type: integer scheduleAlways: description: Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready type: boolean diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 8f0427e1e..9dd4a9c17 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -4282,6 +4282,11 @@ spec: type: array onlyApplyOSDPlacement: type: boolean + osdMaxUpdatesInParallel: + description: The maximum number of OSDs to update in parallel. + format: int32 + minimum: 1 + type: integer scheduleAlways: description: Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready type: boolean diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 22f8fcdef..f4b4f5701 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -3488,6 +3488,10 @@ type StorageScopeSpec struct { // The default is false since data rebalancing can cause temporary cluster slowdown. // +optional AllowOsdCrushWeightUpdate bool `json:"allowOsdCrushWeightUpdate,omitempty"` + // The maximum number of OSDs to update in parallel. + // +kubebuilder:validation:Minimum=1 + // +optional + OSDMaxUpdatesInParallel uint32 `json:"osdMaxUpdatesInParallel,omitempty"` } // Migration handles the OSD migration diff --git a/pkg/daemon/ceph/client/osd.go b/pkg/daemon/ceph/client/osd.go index 49d082e90..4b07de453 100644 --- a/pkg/daemon/ceph/client/osd.go +++ b/pkg/daemon/ceph/client/osd.go @@ -410,7 +410,7 @@ type OSDOkToStopStats struct { // This is relevant, for example, when checking which OSDs can be updated. // The number of OSDs returned is limited by the value set in maxReturned. // maxReturned=0 is the same as maxReturned=1. -func OSDOkToStop(context *clusterd.Context, clusterInfo *ClusterInfo, osdID, maxReturned int) ([]int, error) { +func OSDOkToStop(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int, maxReturned uint32) ([]int, error) { args := []string{"osd", "ok-to-stop", strconv.Itoa(osdID)} // NOTE: if the number of OSD IDs given in the CLI arg query is Q and --max=N is given, if // N < Q, Ceph treats the query as though max=Q instead, always returning at least Q OSDs. diff --git a/pkg/operator/ceph/cluster/osd/update.go b/pkg/operator/ceph/cluster/osd/update.go index fba9dbed7..621140d81 100644 --- a/pkg/operator/ceph/cluster/osd/update.go +++ b/pkg/operator/ceph/cluster/osd/update.go @@ -36,11 +36,11 @@ import ( var ( // allow unit tests to override these values - maxUpdatesInParallel = 20 - updateMultipleDeploymentsAndWaitFunc = k8sutil.UpdateMultipleDeploymentsAndWait - deploymentOnNodeFunc = deploymentOnNode - deploymentOnPVCFunc = deploymentOnPVC - shouldCheckOkToStopFunc = cephclient.OSDUpdateShouldCheckOkToStop + defaultOSDMaxUpdatesInParallel uint32 = 20 + updateMultipleDeploymentsAndWaitFunc = k8sutil.UpdateMultipleDeploymentsAndWait + deploymentOnNodeFunc = deploymentOnNode + deploymentOnPVCFunc = deploymentOnPVC + shouldCheckOkToStopFunc = cephclient.OSDUpdateShouldCheckOkToStop ) type updateConfig struct { @@ -105,7 +105,11 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) { logger.Infof("skipping osd checks for ok-to-stop") osdIDs = []int{osdIDQuery} } else { - osdIDs, err = cephclient.OSDOkToStop(c.cluster.context, c.cluster.clusterInfo, osdIDQuery, maxUpdatesInParallel) + osdMaxUpdatesInParallel := c.cluster.spec.Storage.OSDMaxUpdatesInParallel + if osdMaxUpdatesInParallel == 0 { + osdMaxUpdatesInParallel = defaultOSDMaxUpdatesInParallel + } + osdIDs, err = cephclient.OSDOkToStop(c.cluster.context, c.cluster.clusterInfo, osdIDQuery, osdMaxUpdatesInParallel) if err != nil { if c.cluster.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy { logger.Infof("OSD %d is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing to update it", osdIDQuery)