Merge pull request #16655 from jhoblitt/feature/OSDMaxUpdatesInParallel

osd: add CephCluster.spec.storage.OSDMaxUpdatesInParallel

Documentation/CRDs/Cluster/ceph-cluster-crd.md

@@ -88,6 +88,7 @@ For more details on the mons and when to choose a number other than `3`, see the
     * `allowOsdCrushWeightUpdate`: Whether Rook will resize the OSD CRUSH weight when the OSD PVC size is increased.
         This allows cluster data to be rebalanced to make most effective use of new OSD space.
         The default is false since data rebalancing can cause temporary cluster slowdown.
+    * `osdMaxUpdatesInParallel`: The maximum number of OSDs that are allowed to be simultaneously down during an OSD update. Note that an "update" always takes place upon operator restart and only OSDs which are `ok-to-stop` are taken down. The default value is `20`. Decreasing this value will potentially reduce the impact of updates on the cluster by keeping more OSDs online during an update. Increasing the value may reduce the total time for an update to complete. This is an advanced tuning parameter and the default value should be suitable for most clusters.
     * [storage selection settings](#storage-selection-settings)
     * [Storage Class Device Sets](#storage-class-device-sets)
     * `onlyApplyOSDPlacement`: Whether the placement specific for OSDs is merged with the `all` placement. If `false`, the OSD placement will be merged with the `all` placement. If true, the `OSD placement will be applied` and the `all` placement will be ignored. The placement for OSDs is computed from several different places depending on the type of OSD:

Documentation/CRDs/specification.md

@@ -14624,6 +14624,18 @@ This allows cluster data to be rebalanced to make most effective use of new OSD
 The default is false since data rebalancing can cause temporary cluster slowdown.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>osdMaxUpdatesInParallel</code><br/>
+<em>
+uint32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The maximum number of OSDs to update in parallel.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.StoreType">StoreType

deploy/charts/rook-ceph/templates/resources.yaml

@@ -4284,6 +4284,11 @@ spec:
                       type: array
                     onlyApplyOSDPlacement:
                       type: boolean
+                    osdMaxUpdatesInParallel:
+                      description: The maximum number of OSDs to update in parallel.
+                      format: int32
+                      minimum: 1
+                      type: integer
                     scheduleAlways:
                       description: Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready
                       type: boolean

deploy/examples/crds.yaml

@@ -4282,6 +4282,11 @@ spec:
                       type: array
                     onlyApplyOSDPlacement:
                       type: boolean
+                    osdMaxUpdatesInParallel:
+                      description: The maximum number of OSDs to update in parallel.
+                      format: int32
+                      minimum: 1
+                      type: integer
                     scheduleAlways:
                       description: Whether to always schedule OSDs on a node even if the node is not currently scheduleable or ready
                       type: boolean

pkg/apis/ceph.rook.io/v1/types.go

@@ -3488,6 +3488,10 @@ type StorageScopeSpec struct {
 	// The default is false since data rebalancing can cause temporary cluster slowdown.
 	// +optional
 	AllowOsdCrushWeightUpdate bool `json:"allowOsdCrushWeightUpdate,omitempty"`
+	// The maximum number of OSDs to update in parallel.
+	// +kubebuilder:validation:Minimum=1
+	// +optional
+	OSDMaxUpdatesInParallel uint32 `json:"osdMaxUpdatesInParallel,omitempty"`
 }
 
 // Migration handles the OSD migration

pkg/daemon/ceph/client/osd.go

@@ -410,7 +410,7 @@ type OSDOkToStopStats struct {
 // This is relevant, for example, when checking which OSDs can be updated.
 // The number of OSDs returned is limited by the value set in maxReturned.
 // maxReturned=0 is the same as maxReturned=1.
-func OSDOkToStop(context *clusterd.Context, clusterInfo *ClusterInfo, osdID, maxReturned int) ([]int, error) {
+func OSDOkToStop(context *clusterd.Context, clusterInfo *ClusterInfo, osdID int, maxReturned uint32) ([]int, error) {
 	args := []string{"osd", "ok-to-stop", strconv.Itoa(osdID)}
 	// NOTE: if the number of OSD IDs given in the CLI arg query is Q and --max=N is given, if
 	// N < Q, Ceph treats the query as though max=Q instead, always returning at least Q OSDs.

pkg/operator/ceph/cluster/osd/update.go

@@ -36,11 +36,11 @@ import (
 
 var (
 	// allow unit tests to override these values
-	maxUpdatesInParallel                 = 20
-	updateMultipleDeploymentsAndWaitFunc = k8sutil.UpdateMultipleDeploymentsAndWait
-	deploymentOnNodeFunc                 = deploymentOnNode
-	deploymentOnPVCFunc                  = deploymentOnPVC
-	shouldCheckOkToStopFunc              = cephclient.OSDUpdateShouldCheckOkToStop
+	defaultOSDMaxUpdatesInParallel       uint32 = 20
+	updateMultipleDeploymentsAndWaitFunc        = k8sutil.UpdateMultipleDeploymentsAndWait
+	deploymentOnNodeFunc                        = deploymentOnNode
+	deploymentOnPVCFunc                         = deploymentOnPVC
+	shouldCheckOkToStopFunc                     = cephclient.OSDUpdateShouldCheckOkToStop
 )
 
 type updateConfig struct {
@@ -105,7 +105,11 @@ func (c *updateConfig) updateExistingOSDs(errs *provisionErrors) {
 		logger.Infof("skipping osd checks for ok-to-stop")
 		osdIDs = []int{osdIDQuery}
 	} else {
-		osdIDs, err = cephclient.OSDOkToStop(c.cluster.context, c.cluster.clusterInfo, osdIDQuery, maxUpdatesInParallel)
+		osdMaxUpdatesInParallel := c.cluster.spec.Storage.OSDMaxUpdatesInParallel
+		if osdMaxUpdatesInParallel == 0 {
+			osdMaxUpdatesInParallel = defaultOSDMaxUpdatesInParallel
+		}
+		osdIDs, err = cephclient.OSDOkToStop(c.cluster.context, c.cluster.clusterInfo, osdIDQuery, osdMaxUpdatesInParallel)
 		if err != nil {
 			if c.cluster.spec.ContinueUpgradeAfterChecksEvenIfNotHealthy {
 				logger.Infof("OSD %d is not ok-to-stop but 'continueUpgradeAfterChecksEvenIfNotHealthy' is true, so continuing to update it", osdIDQuery)