From 729322ffd7ab341dacd35d094af8ff9cc95a10c2 Mon Sep 17 00:00:00 2001 From: Tim Buchwaldt Date: Mon, 6 Oct 2025 10:57:49 +0200 Subject: [PATCH] operator: allow overriding MDS cache memory limit This change introduces support for setting the ceph cache memory limit ratios that were previously hardcoded. Codegen was drivey-by updated as the current version errored. Resolves #16551 Signed-off-by: Tim Buchwaldt (cherry picked from commit 8db5274703c2bc0d0246ac4ef219c0bff9841637) --- Documentation/CRDs/specification.md | 28 ++ .../charts/rook-ceph/templates/resources.yaml | 16 ++ deploy/examples/crds.yaml | 16 ++ deploy/examples/filesystem.yaml | 9 + pkg/apis/ceph.rook.io/v1/types.go | 16 ++ .../ceph.rook.io/v1/zz_generated.deepcopy.go | 10 + pkg/operator/ceph/file/mds/config.go | 14 +- pkg/operator/ceph/file/mds/config_test.go | 246 ++++++++++++++++++ pkg/operator/ceph/file/mds/spec.go | 4 +- 9 files changed, 355 insertions(+), 4 deletions(-) create mode 100644 pkg/operator/ceph/file/mds/config_test.go diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 282a11dcb..933d29d34 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -9029,6 +9029,34 @@ ProbeSpec (Optional) + + +cacheMemoryLimitFactor
+ +float64 + + + +(Optional) +

CacheMemoryLimitFactor is the factor applied to the memory limit to determine the MDS cache memory limit. +MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container. +MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM. +This factor is applied when resources.limits.memory is set.

+ + + + +cacheMemoryRequestFactor
+ +float64 + + + +(Optional) +

CacheMemoryRequestFactor is the factor applied to the memory request to determine the MDS cache memory limit. +This factor is applied when resources.requests.memory is set and resources.limits.memory is not set.

+ +

MgrSpec diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 461a2e33e..031d95908 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -8062,6 +8062,22 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + cacheMemoryLimitFactor: + description: |- + CacheMemoryLimitFactor is the factor applied to the memory limit to determine the MDS cache memory limit. + MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container. + MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM. + This factor is applied when resources.limits.memory is set. + maximum: 1 + minimum: 0 + type: number + cacheMemoryRequestFactor: + description: |- + CacheMemoryRequestFactor is the factor applied to the memory request to determine the MDS cache memory limit. + This factor is applied when resources.requests.memory is set and resources.limits.memory is not set. + maximum: 1 + minimum: 0 + type: number labels: additionalProperties: type: string diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index d64d50788..6e3e38bda 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -8057,6 +8057,22 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + cacheMemoryLimitFactor: + description: |- + CacheMemoryLimitFactor is the factor applied to the memory limit to determine the MDS cache memory limit. + MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container. + MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM. + This factor is applied when resources.limits.memory is set. + maximum: 1 + minimum: 0 + type: number + cacheMemoryRequestFactor: + description: |- + CacheMemoryRequestFactor is the factor applied to the memory request to determine the MDS cache memory limit. + This factor is applied when resources.requests.memory is set and resources.limits.memory is not set. + maximum: 1 + minimum: 0 + type: number labels: additionalProperties: type: string diff --git a/deploy/examples/filesystem.yaml b/deploy/examples/filesystem.yaml index 740fcaf1a..6f12f5fe3 100644 --- a/deploy/examples/filesystem.yaml +++ b/deploy/examples/filesystem.yaml @@ -105,6 +105,15 @@ spec: # requests: # cpu: "500m" # memory: "1024Mi" + # Factor applied to resources.limits.memory to determine the MDS cache memory limit. + # Default is 0.5 (50% of memory limit). MDS uses approximately 125% of this value in RAM. + # For high-volume file ingest/delete scenarios, consider lower values like 0.25. + # Valid range: 0.0 - 1.0 + # cacheMemoryLimitFactor: 0.5 + # Factor applied to resources.requests.memory when limits.memory is not set. + # Default is 0.8 (80% of memory request). + # Valid range: 0.0 - 1.0 + # cacheMemoryRequestFactor: 0.8 priorityClassName: system-cluster-critical livenessProbe: disabled: false diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 5d52d32f2..a1aed4aea 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1448,6 +1448,22 @@ type MetadataServerSpec struct { // +optional StartupProbe *ProbeSpec `json:"startupProbe,omitempty"` + + // CacheMemoryLimitFactor is the factor applied to the memory limit to determine the MDS cache memory limit. + // MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container. + // MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM. + // This factor is applied when resources.limits.memory is set. + // +kubebuilder:validation:Minimum=0.0 + // +kubebuilder:validation:Maximum=1.0 + // +optional + CacheMemoryLimitFactor *float64 `json:"cacheMemoryLimitFactor,omitempty"` + + // CacheMemoryRequestFactor is the factor applied to the memory request to determine the MDS cache memory limit. + // This factor is applied when resources.requests.memory is set and resources.limits.memory is not set. + // +kubebuilder:validation:Minimum=0.0 + // +kubebuilder:validation:Maximum=1.0 + // +optional + CacheMemoryRequestFactor *float64 `json:"cacheMemoryRequestFactor,omitempty"` } // FSMirroringSpec represents the setting for a mirrored filesystem diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 92a52dded..44f197742 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -3307,6 +3307,16 @@ func (in *MetadataServerSpec) DeepCopyInto(out *MetadataServerSpec) { *out = new(ProbeSpec) (*in).DeepCopyInto(*out) } + if in.CacheMemoryLimitFactor != nil { + in, out := &in.CacheMemoryLimitFactor, &out.CacheMemoryLimitFactor + *out = new(float64) + **out = **in + } + if in.CacheMemoryRequestFactor != nil { + in, out := &in.CacheMemoryRequestFactor, &out.CacheMemoryRequestFactor + *out = new(float64) + **out = **in + } return } diff --git a/pkg/operator/ceph/file/mds/config.go b/pkg/operator/ceph/file/mds/config.go index 601d0655d..6cb74ca98 100644 --- a/pkg/operator/ceph/file/mds/config.go +++ b/pkg/operator/ceph/file/mds/config.go @@ -80,10 +80,20 @@ func (c *Cluster) setDefaultFlagsMonConfigStore(mdsID string) error { // Set mds cache memory limit to the best appropriate value if !c.fs.Spec.MetadataServer.Resources.Limits.Memory().IsZero() { - mdsCacheMemoryLimit := float64(c.fs.Spec.MetadataServer.Resources.Limits.Memory().Value()) * mdsCacheMemoryLimitFactor + // Use configured factor or fall back to default + limitFactor := mdsCacheMemoryLimitFactor + if c.fs.Spec.MetadataServer.CacheMemoryLimitFactor != nil { + limitFactor = *c.fs.Spec.MetadataServer.CacheMemoryLimitFactor + } + mdsCacheMemoryLimit := float64(c.fs.Spec.MetadataServer.Resources.Limits.Memory().Value()) * limitFactor configOptions["mds_cache_memory_limit"] = strconv.Itoa(int(mdsCacheMemoryLimit)) } else if !c.fs.Spec.MetadataServer.Resources.Requests.Memory().IsZero() { - mdsCacheMemoryRequest := float64(c.fs.Spec.MetadataServer.Resources.Requests.Memory().Value()) * mdsCacheMemoryResourceFactor + // Use configured factor or fall back to default + requestFactor := mdsCacheMemoryRequestFactor + if c.fs.Spec.MetadataServer.CacheMemoryRequestFactor != nil { + requestFactor = *c.fs.Spec.MetadataServer.CacheMemoryRequestFactor + } + mdsCacheMemoryRequest := float64(c.fs.Spec.MetadataServer.Resources.Requests.Memory().Value()) * requestFactor configOptions["mds_cache_memory_limit"] = strconv.Itoa(int(mdsCacheMemoryRequest)) } diff --git a/pkg/operator/ceph/file/mds/config_test.go b/pkg/operator/ceph/file/mds/config_test.go new file mode 100644 index 000000000..d187c7435 --- /dev/null +++ b/pkg/operator/ceph/file/mds/config_test.go @@ -0,0 +1,246 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mds + +import ( + "context" + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestSetDefaultFlagsMonConfigStore(t *testing.T) { + clusterInfo := &cephclient.ClusterInfo{ + Context: context.TODO(), + Namespace: "ns", + FSID: "myfsid", + } + + executor := &exectest.MockExecutor{} + ctx := &clusterd.Context{ + Executor: executor, + } + + t.Run("default factors with memory limit", func(t *testing.T) { + memoryLimit := resource.MustParse("1Gi") + fs := cephv1.CephFilesystem{ + ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"}, + Spec: cephv1.FilesystemSpec{ + MetadataServer: cephv1.MetadataServerSpec{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memoryLimit, + }, + }, + }, + }, + } + + c := &Cluster{ + clusterInfo: clusterInfo, + context: ctx, + fs: fs, + } + + // Mock config set command + expectedCacheLimit := int(float64(memoryLimit.Value()) * mdsCacheMemoryLimitFactor) + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_cache_memory_limit" { + assert.Equal(t, "mds.myfs-a", args[2]) + assert.Equal(t, "536870912", args[4]) // 1Gi * 0.5 + return "", nil + } + if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" { + assert.Equal(t, "mds.myfs-a", args[2]) + assert.Equal(t, "myfs", args[4]) + return "", nil + } + return "", nil + } + + err := c.setDefaultFlagsMonConfigStore("myfs-a") + assert.NoError(t, err) + assert.Equal(t, 536870912, expectedCacheLimit) + }) + + t.Run("custom limit factor", func(t *testing.T) { + memoryLimit := resource.MustParse("1Gi") + customFactor := 0.25 + fs := cephv1.CephFilesystem{ + ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"}, + Spec: cephv1.FilesystemSpec{ + MetadataServer: cephv1.MetadataServerSpec{ + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: memoryLimit, + }, + }, + CacheMemoryLimitFactor: &customFactor, + }, + }, + } + + c := &Cluster{ + clusterInfo: clusterInfo, + context: ctx, + fs: fs, + } + + // Mock config set command + expectedCacheLimit := int(float64(memoryLimit.Value()) * customFactor) + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_cache_memory_limit" { + assert.Equal(t, "mds.myfs-a", args[2]) + assert.Equal(t, "268435456", args[4]) // 1Gi * 0.25 + return "", nil + } + if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" { + assert.Equal(t, "mds.myfs-a", args[2]) + assert.Equal(t, "myfs", args[4]) + return "", nil + } + return "", nil + } + + err := c.setDefaultFlagsMonConfigStore("myfs-a") + assert.NoError(t, err) + assert.Equal(t, 268435456, expectedCacheLimit) + }) + + t.Run("default factors with memory request", func(t *testing.T) { + memoryRequest := resource.MustParse("512Mi") + fs := cephv1.CephFilesystem{ + ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"}, + Spec: cephv1.FilesystemSpec{ + MetadataServer: cephv1.MetadataServerSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: memoryRequest, + }, + }, + }, + }, + } + + c := &Cluster{ + clusterInfo: clusterInfo, + context: ctx, + fs: fs, + } + + // Mock config set command + expectedCacheLimit := int(float64(memoryRequest.Value()) * mdsCacheMemoryRequestFactor) + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_cache_memory_limit" { + assert.Equal(t, "mds.myfs-a", args[2]) + assert.Equal(t, "429496729", args[4]) // 512Mi * 0.8 + return "", nil + } + if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" { + assert.Equal(t, "mds.myfs-a", args[2]) + assert.Equal(t, "myfs", args[4]) + return "", nil + } + return "", nil + } + + err := c.setDefaultFlagsMonConfigStore("myfs-a") + assert.NoError(t, err) + assert.Equal(t, 429496729, expectedCacheLimit) + }) + + t.Run("custom request factor", func(t *testing.T) { + memoryRequest := resource.MustParse("512Mi") + customFactor := 0.6 + fs := cephv1.CephFilesystem{ + ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"}, + Spec: cephv1.FilesystemSpec{ + MetadataServer: cephv1.MetadataServerSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: memoryRequest, + }, + }, + CacheMemoryRequestFactor: &customFactor, + }, + }, + } + + c := &Cluster{ + clusterInfo: clusterInfo, + context: ctx, + fs: fs, + } + + // Mock config set command + expectedCacheLimit := int(float64(memoryRequest.Value()) * customFactor) + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_cache_memory_limit" { + assert.Equal(t, "mds.myfs-a", args[2]) + assert.Equal(t, "322122547", args[4]) // 512Mi * 0.6 + return "", nil + } + if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" { + assert.Equal(t, "mds.myfs-a", args[2]) + assert.Equal(t, "myfs", args[4]) + return "", nil + } + return "", nil + } + + err := c.setDefaultFlagsMonConfigStore("myfs-a") + assert.NoError(t, err) + assert.Equal(t, 322122547, expectedCacheLimit) + }) + + t.Run("no memory specified", func(t *testing.T) { + fs := cephv1.CephFilesystem{ + ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"}, + Spec: cephv1.FilesystemSpec{ + MetadataServer: cephv1.MetadataServerSpec{ + Resources: v1.ResourceRequirements{}, + }, + }, + } + + c := &Cluster{ + clusterInfo: clusterInfo, + context: ctx, + fs: fs, + } + + // Mock config set command - should only set mds_join_fs + executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) { + if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" { + assert.Equal(t, "mds.myfs-a", args[2]) + assert.Equal(t, "myfs", args[4]) + return "", nil + } + return "", nil + } + + err := c.setDefaultFlagsMonConfigStore("myfs-a") + assert.NoError(t, err) + }) +} diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index febbf98ed..3deb14a34 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -38,8 +38,8 @@ const ( // MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container // MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM. // Eventually we will tune this automatically: http://tracker.ceph.com/issues/36663 - mdsCacheMemoryLimitFactor = 0.5 - mdsCacheMemoryResourceFactor = 0.8 + mdsCacheMemoryLimitFactor = 0.5 + mdsCacheMemoryRequestFactor = 0.8 ) func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, fsNamespacedname types.NamespacedName) (*apps.Deployment, error) {