operator: allow overriding MDS cache memory limit

This change introduces support for setting the ceph cache memory limit
ratios that were previously hardcoded.
Codegen was drivey-by updated as the current version errored.

Resolves #16551

Signed-off-by: Tim Buchwaldt <tim.buchwaldt@deepl.com>
(cherry picked from commit 8db5274703)

Documentation/CRDs/specification.md

@@ -9029,6 +9029,34 @@ ProbeSpec
 <em>(Optional)</em>
 </td>
 </tr>
+<tr>
+<td>
+<code>cacheMemoryLimitFactor</code><br/>
+<em>
+float64
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CacheMemoryLimitFactor is the factor applied to the memory limit to determine the MDS cache memory limit.
+MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container.
+MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM.
+This factor is applied when resources.limits.memory is set.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>cacheMemoryRequestFactor</code><br/>
+<em>
+float64
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CacheMemoryRequestFactor is the factor applied to the memory request to determine the MDS cache memory limit.
+This factor is applied when resources.requests.memory is set and resources.limits.memory is not set.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.MgrSpec">MgrSpec

deploy/charts/rook-ceph/templates/resources.yaml

@@ -8062,6 +8062,22 @@ spec:
                       nullable: true
                       type: object
                       x-kubernetes-preserve-unknown-fields: true
+                    cacheMemoryLimitFactor:
+                      description: |-
+                        CacheMemoryLimitFactor is the factor applied to the memory limit to determine the MDS cache memory limit.
+                        MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container.
+                        MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM.
+                        This factor is applied when resources.limits.memory is set.
+                      maximum: 1
+                      minimum: 0
+                      type: number
+                    cacheMemoryRequestFactor:
+                      description: |-
+                        CacheMemoryRequestFactor is the factor applied to the memory request to determine the MDS cache memory limit.
+                        This factor is applied when resources.requests.memory is set and resources.limits.memory is not set.
+                      maximum: 1
+                      minimum: 0
+                      type: number
                     labels:
                       additionalProperties:
                         type: string

deploy/examples/crds.yaml

@@ -8057,6 +8057,22 @@ spec:
                       nullable: true
                       type: object
                       x-kubernetes-preserve-unknown-fields: true
+                    cacheMemoryLimitFactor:
+                      description: |-
+                        CacheMemoryLimitFactor is the factor applied to the memory limit to determine the MDS cache memory limit.
+                        MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container.
+                        MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM.
+                        This factor is applied when resources.limits.memory is set.
+                      maximum: 1
+                      minimum: 0
+                      type: number
+                    cacheMemoryRequestFactor:
+                      description: |-
+                        CacheMemoryRequestFactor is the factor applied to the memory request to determine the MDS cache memory limit.
+                        This factor is applied when resources.requests.memory is set and resources.limits.memory is not set.
+                      maximum: 1
+                      minimum: 0
+                      type: number
                     labels:
                       additionalProperties:
                         type: string

deploy/examples/filesystem.yaml

@@ -105,6 +105,15 @@ spec:
     #  requests:
     #    cpu: "500m"
     #    memory: "1024Mi"
+    # Factor applied to resources.limits.memory to determine the MDS cache memory limit.
+    # Default is 0.5 (50% of memory limit). MDS uses approximately 125% of this value in RAM.
+    # For high-volume file ingest/delete scenarios, consider lower values like 0.25.
+    # Valid range: 0.0 - 1.0
+    # cacheMemoryLimitFactor: 0.5
+    # Factor applied to resources.requests.memory when limits.memory is not set.
+    # Default is 0.8 (80% of memory request).
+    # Valid range: 0.0 - 1.0
+    # cacheMemoryRequestFactor: 0.8
     priorityClassName: system-cluster-critical
     livenessProbe:
       disabled: false

pkg/apis/ceph.rook.io/v1/types.go

@@ -1448,6 +1448,22 @@ type MetadataServerSpec struct {
 
 	// +optional
 	StartupProbe *ProbeSpec `json:"startupProbe,omitempty"`
+
+	// CacheMemoryLimitFactor is the factor applied to the memory limit to determine the MDS cache memory limit.
+	// MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container.
+	// MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM.
+	// This factor is applied when resources.limits.memory is set.
+	// +kubebuilder:validation:Minimum=0.0
+	// +kubebuilder:validation:Maximum=1.0
+	// +optional
+	CacheMemoryLimitFactor *float64 `json:"cacheMemoryLimitFactor,omitempty"`
+
+	// CacheMemoryRequestFactor is the factor applied to the memory request to determine the MDS cache memory limit.
+	// This factor is applied when resources.requests.memory is set and resources.limits.memory is not set.
+	// +kubebuilder:validation:Minimum=0.0
+	// +kubebuilder:validation:Maximum=1.0
+	// +optional
+	CacheMemoryRequestFactor *float64 `json:"cacheMemoryRequestFactor,omitempty"`
 }
 
 // FSMirroringSpec represents the setting for a mirrored filesystem

pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go

@@ -3307,6 +3307,16 @@ func (in *MetadataServerSpec) DeepCopyInto(out *MetadataServerSpec) {
 		*out = new(ProbeSpec)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.CacheMemoryLimitFactor != nil {
+		in, out := &in.CacheMemoryLimitFactor, &out.CacheMemoryLimitFactor
+		*out = new(float64)
+		**out = **in
+	}
+	if in.CacheMemoryRequestFactor != nil {
+		in, out := &in.CacheMemoryRequestFactor, &out.CacheMemoryRequestFactor
+		*out = new(float64)
+		**out = **in
+	}
 	return
 }
 

pkg/operator/ceph/file/mds/config.go

@@ -80,10 +80,20 @@ func (c *Cluster) setDefaultFlagsMonConfigStore(mdsID string) error {
 
 	// Set mds cache memory limit to the best appropriate value
 	if !c.fs.Spec.MetadataServer.Resources.Limits.Memory().IsZero() {
-		mdsCacheMemoryLimit := float64(c.fs.Spec.MetadataServer.Resources.Limits.Memory().Value()) * mdsCacheMemoryLimitFactor
+		// Use configured factor or fall back to default
+		limitFactor := mdsCacheMemoryLimitFactor
+		if c.fs.Spec.MetadataServer.CacheMemoryLimitFactor != nil {
+			limitFactor = *c.fs.Spec.MetadataServer.CacheMemoryLimitFactor
+		}
+		mdsCacheMemoryLimit := float64(c.fs.Spec.MetadataServer.Resources.Limits.Memory().Value()) * limitFactor
 		configOptions["mds_cache_memory_limit"] = strconv.Itoa(int(mdsCacheMemoryLimit))
 	} else if !c.fs.Spec.MetadataServer.Resources.Requests.Memory().IsZero() {
-		mdsCacheMemoryRequest := float64(c.fs.Spec.MetadataServer.Resources.Requests.Memory().Value()) * mdsCacheMemoryResourceFactor
+		// Use configured factor or fall back to default
+		requestFactor := mdsCacheMemoryRequestFactor
+		if c.fs.Spec.MetadataServer.CacheMemoryRequestFactor != nil {
+			requestFactor = *c.fs.Spec.MetadataServer.CacheMemoryRequestFactor
+		}
+		mdsCacheMemoryRequest := float64(c.fs.Spec.MetadataServer.Resources.Requests.Memory().Value()) * requestFactor
 		configOptions["mds_cache_memory_limit"] = strconv.Itoa(int(mdsCacheMemoryRequest))
 	}
 

pkg/operator/ceph/file/mds/config_test.go

@@ -0,0 +1,246 @@
+/*
+Copyright 2025 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mds
+
+import (
+	"context"
+	"testing"
+
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"github.com/rook/rook/pkg/clusterd"
+	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+	exectest "github.com/rook/rook/pkg/util/exec/test"
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestSetDefaultFlagsMonConfigStore(t *testing.T) {
+	clusterInfo := &cephclient.ClusterInfo{
+		Context:   context.TODO(),
+		Namespace: "ns",
+		FSID:      "myfsid",
+	}
+
+	executor := &exectest.MockExecutor{}
+	ctx := &clusterd.Context{
+		Executor: executor,
+	}
+
+	t.Run("default factors with memory limit", func(t *testing.T) {
+		memoryLimit := resource.MustParse("1Gi")
+		fs := cephv1.CephFilesystem{
+			ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"},
+			Spec: cephv1.FilesystemSpec{
+				MetadataServer: cephv1.MetadataServerSpec{
+					Resources: v1.ResourceRequirements{
+						Limits: v1.ResourceList{
+							v1.ResourceMemory: memoryLimit,
+						},
+					},
+				},
+			},
+		}
+
+		c := &Cluster{
+			clusterInfo: clusterInfo,
+			context:     ctx,
+			fs:          fs,
+		}
+
+		// Mock config set command
+		expectedCacheLimit := int(float64(memoryLimit.Value()) * mdsCacheMemoryLimitFactor)
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_cache_memory_limit" {
+				assert.Equal(t, "mds.myfs-a", args[2])
+				assert.Equal(t, "536870912", args[4]) // 1Gi * 0.5
+				return "", nil
+			}
+			if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" {
+				assert.Equal(t, "mds.myfs-a", args[2])
+				assert.Equal(t, "myfs", args[4])
+				return "", nil
+			}
+			return "", nil
+		}
+
+		err := c.setDefaultFlagsMonConfigStore("myfs-a")
+		assert.NoError(t, err)
+		assert.Equal(t, 536870912, expectedCacheLimit)
+	})
+
+	t.Run("custom limit factor", func(t *testing.T) {
+		memoryLimit := resource.MustParse("1Gi")
+		customFactor := 0.25
+		fs := cephv1.CephFilesystem{
+			ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"},
+			Spec: cephv1.FilesystemSpec{
+				MetadataServer: cephv1.MetadataServerSpec{
+					Resources: v1.ResourceRequirements{
+						Limits: v1.ResourceList{
+							v1.ResourceMemory: memoryLimit,
+						},
+					},
+					CacheMemoryLimitFactor: &customFactor,
+				},
+			},
+		}
+
+		c := &Cluster{
+			clusterInfo: clusterInfo,
+			context:     ctx,
+			fs:          fs,
+		}
+
+		// Mock config set command
+		expectedCacheLimit := int(float64(memoryLimit.Value()) * customFactor)
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_cache_memory_limit" {
+				assert.Equal(t, "mds.myfs-a", args[2])
+				assert.Equal(t, "268435456", args[4]) // 1Gi * 0.25
+				return "", nil
+			}
+			if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" {
+				assert.Equal(t, "mds.myfs-a", args[2])
+				assert.Equal(t, "myfs", args[4])
+				return "", nil
+			}
+			return "", nil
+		}
+
+		err := c.setDefaultFlagsMonConfigStore("myfs-a")
+		assert.NoError(t, err)
+		assert.Equal(t, 268435456, expectedCacheLimit)
+	})
+
+	t.Run("default factors with memory request", func(t *testing.T) {
+		memoryRequest := resource.MustParse("512Mi")
+		fs := cephv1.CephFilesystem{
+			ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"},
+			Spec: cephv1.FilesystemSpec{
+				MetadataServer: cephv1.MetadataServerSpec{
+					Resources: v1.ResourceRequirements{
+						Requests: v1.ResourceList{
+							v1.ResourceMemory: memoryRequest,
+						},
+					},
+				},
+			},
+		}
+
+		c := &Cluster{
+			clusterInfo: clusterInfo,
+			context:     ctx,
+			fs:          fs,
+		}
+
+		// Mock config set command
+		expectedCacheLimit := int(float64(memoryRequest.Value()) * mdsCacheMemoryRequestFactor)
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_cache_memory_limit" {
+				assert.Equal(t, "mds.myfs-a", args[2])
+				assert.Equal(t, "429496729", args[4]) // 512Mi * 0.8
+				return "", nil
+			}
+			if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" {
+				assert.Equal(t, "mds.myfs-a", args[2])
+				assert.Equal(t, "myfs", args[4])
+				return "", nil
+			}
+			return "", nil
+		}
+
+		err := c.setDefaultFlagsMonConfigStore("myfs-a")
+		assert.NoError(t, err)
+		assert.Equal(t, 429496729, expectedCacheLimit)
+	})
+
+	t.Run("custom request factor", func(t *testing.T) {
+		memoryRequest := resource.MustParse("512Mi")
+		customFactor := 0.6
+		fs := cephv1.CephFilesystem{
+			ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"},
+			Spec: cephv1.FilesystemSpec{
+				MetadataServer: cephv1.MetadataServerSpec{
+					Resources: v1.ResourceRequirements{
+						Requests: v1.ResourceList{
+							v1.ResourceMemory: memoryRequest,
+						},
+					},
+					CacheMemoryRequestFactor: &customFactor,
+				},
+			},
+		}
+
+		c := &Cluster{
+			clusterInfo: clusterInfo,
+			context:     ctx,
+			fs:          fs,
+		}
+
+		// Mock config set command
+		expectedCacheLimit := int(float64(memoryRequest.Value()) * customFactor)
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_cache_memory_limit" {
+				assert.Equal(t, "mds.myfs-a", args[2])
+				assert.Equal(t, "322122547", args[4]) // 512Mi * 0.6
+				return "", nil
+			}
+			if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" {
+				assert.Equal(t, "mds.myfs-a", args[2])
+				assert.Equal(t, "myfs", args[4])
+				return "", nil
+			}
+			return "", nil
+		}
+
+		err := c.setDefaultFlagsMonConfigStore("myfs-a")
+		assert.NoError(t, err)
+		assert.Equal(t, 322122547, expectedCacheLimit)
+	})
+
+	t.Run("no memory specified", func(t *testing.T) {
+		fs := cephv1.CephFilesystem{
+			ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"},
+			Spec: cephv1.FilesystemSpec{
+				MetadataServer: cephv1.MetadataServerSpec{
+					Resources: v1.ResourceRequirements{},
+				},
+			},
+		}
+
+		c := &Cluster{
+			clusterInfo: clusterInfo,
+			context:     ctx,
+			fs:          fs,
+		}
+
+		// Mock config set command - should only set mds_join_fs
+		executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
+			if len(args) >= 4 && args[0] == "config" && args[1] == "set" && args[3] == "mds_join_fs" {
+				assert.Equal(t, "mds.myfs-a", args[2])
+				assert.Equal(t, "myfs", args[4])
+				return "", nil
+			}
+			return "", nil
+		}
+
+		err := c.setDefaultFlagsMonConfigStore("myfs-a")
+		assert.NoError(t, err)
+	})
+}

pkg/operator/ceph/file/mds/spec.go

@@ -38,8 +38,8 @@ const (
 	// MDS cache memory limit should be set to 50-60% of RAM reserved for the MDS container
 	// MDS uses approximately 125% of the value of mds_cache_memory_limit in RAM.
 	// Eventually we will tune this automatically: http://tracker.ceph.com/issues/36663
-	mdsCacheMemoryLimitFactor    = 0.5
-	mdsCacheMemoryResourceFactor = 0.8
+	mdsCacheMemoryLimitFactor   = 0.5
+	mdsCacheMemoryRequestFactor = 0.8
 )
 
 func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, fsNamespacedname types.NamespacedName) (*apps.Deployment, error) {