From d71f9c238fa4bb73f175639b2b552d837ae45055 Mon Sep 17 00:00:00 2001 From: Deepika Upadhyay Date: Fri, 7 Jun 2024 18:10:18 +0530 Subject: [PATCH] pool: add option to enableCrushUpdates to pool if required this will be disabled by default but in scenarios where the user want to update failureDomain, DeviceClass etc, this option can be enabled, to be noted this can lead to lot of data rebalancing and remapping. Use with caution Signed-off-by: Deepika Upadhyay --- Documentation/CRDs/specification.md | 12 +++++++++ .../charts/rook-ceph/templates/resources.yaml | 21 +++++++++++++++ deploy/examples/crds.yaml | 21 +++++++++++++++ deploy/examples/pool.yaml | 2 ++ pkg/apis/ceph.rook.io/v1/types.go | 4 +++ pkg/daemon/ceph/client/pool.go | 6 +++++ pkg/daemon/ceph/client/pool_test.go | 27 +++++++++++++++---- 7 files changed, 88 insertions(+), 5 deletions(-) diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index f03bc8648..a16592e14 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -10478,6 +10478,18 @@ string +enableCrushUpdates
+ +bool + + + +(Optional) +

Allow rook operator to change the pool CRUSH tunables once the pool is created

+ + + + compressionMode
string diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index bac4fb8a1..da00db828 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -184,6 +184,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -6540,6 +6543,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -6721,6 +6727,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -10198,6 +10207,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -11310,6 +11322,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -12037,6 +12052,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -12213,6 +12231,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 0a510790c..fe48be2f6 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -186,6 +186,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -6535,6 +6538,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -6716,6 +6722,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -10189,6 +10198,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -11301,6 +11313,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -12025,6 +12040,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean @@ -12201,6 +12219,9 @@ spec: description: The device class the OSD should set to for use in the pool nullable: true type: string + enableCrushUpdates: + description: Allow rook operator to change the pool CRUSH tunables once the pool is created + type: boolean enableRBDStats: description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool type: boolean diff --git a/deploy/examples/pool.yaml b/deploy/examples/pool.yaml index aa1668ae2..8316dacf3 100644 --- a/deploy/examples/pool.yaml +++ b/deploy/examples/pool.yaml @@ -33,6 +33,8 @@ spec: # If device classes are specified, ensure this property is added to every pool in the cluster, # otherwise Ceph will warn about pools with overlapping roots. #deviceClass: my-class + # allow rook to update pool spec and crush rule, use with caution, disabled by default + #enableCrushUpdates: false # Enables collecting RBD per-image IO statistics by enabling dynamic OSD performance counters. Defaults to false. # For reference: https://docs.ceph.com/docs/master/mgr/prometheus/#rbd-io-statistics # enableRBDStats: true diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index f305f9553..d55ae3ccf 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -755,6 +755,10 @@ type PoolSpec struct { // +nullable DeviceClass string `json:"deviceClass,omitempty"` + // Allow rook operator to change the pool CRUSH tunables once the pool is created + // +optional + EnableCrushUpdates bool `json:"enableCrushUpdates,omitempty"` + // DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" // The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) // +kubebuilder:validation:Enum=none;passive;aggressive;force;"" diff --git a/pkg/daemon/ceph/client/pool.go b/pkg/daemon/ceph/client/pool.go index b7089f94b..65e2f87df 100644 --- a/pkg/daemon/ceph/client/pool.go +++ b/pkg/daemon/ceph/client/pool.go @@ -463,6 +463,12 @@ func createReplicatedPoolForApp(context *clusterd.Context, clusterInfo *ClusterI } func updatePoolCrushRule(context *clusterd.Context, clusterInfo *ClusterInfo, clusterSpec *cephv1.ClusterSpec, pool cephv1.NamedPoolSpec) error { + + if !pool.EnableCrushUpdates { + logger.Debugf("Skipping crush rule update for pool %q: EnableCrushUpdates is disabled", pool.Name) + return nil + } + if pool.FailureDomain == "" && pool.DeviceClass == "" { logger.Debugf("skipping check for failure domain and deviceClass on pool %q as it is not specified", pool.Name) return nil diff --git a/pkg/daemon/ceph/client/pool_test.go b/pkg/daemon/ceph/client/pool_test.go index b2fa1add2..bceabb918 100644 --- a/pkg/daemon/ceph/client/pool_test.go +++ b/pkg/daemon/ceph/client/pool_test.go @@ -281,7 +281,8 @@ func TestUpdateFailureDomain(t *testing.T) { p := cephv1.NamedPoolSpec{ Name: "mypool", PoolSpec: cephv1.PoolSpec{ - Replicated: cephv1.ReplicatedSpec{Size: 3}, + Replicated: cephv1.ReplicatedSpec{Size: 3}, + EnableCrushUpdates: true, }, } clusterSpec := &cephv1.ClusterSpec{Storage: cephv1.StorageScopeSpec{}} @@ -294,8 +295,9 @@ func TestUpdateFailureDomain(t *testing.T) { p := cephv1.NamedPoolSpec{ Name: "mypool", PoolSpec: cephv1.PoolSpec{ - FailureDomain: currentFailureDomain, - Replicated: cephv1.ReplicatedSpec{Size: 3}, + FailureDomain: currentFailureDomain, + Replicated: cephv1.ReplicatedSpec{Size: 3}, + EnableCrushUpdates: true, }, } testCrushRuleName = "mypool_rack" @@ -305,12 +307,27 @@ func TestUpdateFailureDomain(t *testing.T) { assert.Equal(t, "", newCrushRule) }) + t.Run("trying to change failure domain without enabling EnableCrushUpdates", func(t *testing.T) { + p := cephv1.NamedPoolSpec{ + Name: "mypool", + PoolSpec: cephv1.PoolSpec{ + FailureDomain: "zone", + Replicated: cephv1.ReplicatedSpec{Size: 3}, + }, + } + clusterSpec := &cephv1.ClusterSpec{Storage: cephv1.StorageScopeSpec{}} + err := updatePoolCrushRule(context, AdminTestClusterInfo("mycluster"), clusterSpec, p) + assert.NoError(t, err) + assert.Equal(t, "", newCrushRule) + }) + t.Run("changing failure domain", func(t *testing.T) { p := cephv1.NamedPoolSpec{ Name: "mypool", PoolSpec: cephv1.PoolSpec{ - FailureDomain: "zone", - Replicated: cephv1.ReplicatedSpec{Size: 3}, + FailureDomain: "zone", + Replicated: cephv1.ReplicatedSpec{Size: 3}, + EnableCrushUpdates: true, }, } clusterSpec := &cephv1.ClusterSpec{Storage: cephv1.StorageScopeSpec{}}