From bc6e5fbc8693c6a253071603facd553dec7ae940 Mon Sep 17 00:00:00 2001 From: Oded Viner Date: Mon, 16 Feb 2026 14:31:35 +0200 Subject: [PATCH] nvmeof: add default gateway pod spreading ensure each gw pod runs on a different node when possible. add default pod anti-affinity and topology spread constraints. keep user placement overrides by merging defaults with spec placement. add tests for default spread and user override behavior. Signed-off-by: Oded Viner --- pkg/operator/ceph/nvmeof/spec.go | 20 +++++- pkg/operator/ceph/nvmeof/spec_test.go | 89 +++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/pkg/operator/ceph/nvmeof/spec.go b/pkg/operator/ceph/nvmeof/spec.go index 06b415927..02b3d6622 100644 --- a/pkg/operator/ceph/nvmeof/spec.go +++ b/pkg/operator/ceph/nvmeof/spec.go @@ -217,7 +217,8 @@ func (r *ReconcileCephNVMeOFGateway) makeDeployment(nvmeof *cephv1.CephNVMeOFGat if hostNetwork { podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet } - nvmeof.Spec.Placement.ApplyToPodSpec(&podSpec) + // Apply default spreading for gateway pods and let user-provided placement override defaults. + getDefaultNVMeOFPlacement().Merge(nvmeof.Spec.Placement).ApplyToPodSpec(&podSpec) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -254,6 +255,23 @@ func (r *ReconcileCephNVMeOFGateway) makeDeployment(nvmeof *cephv1.CephNVMeOFGat return deployment, nil } +func getDefaultNVMeOFPlacement() cephv1.Placement { + return cephv1.Placement{ + TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "kubernetes.io/hostname", + WhenUnsatisfiable: v1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": AppName, + }, + }, + }, + }, + } +} + // createCephConfigInitContainer creates a minimal init container that generates // /etc/ceph/ceph.conf, copies the admin keyring, and copies nvmeof.conf from the ConfigMap. // This is needed for the gateway to connect to the Ceph cluster. diff --git a/pkg/operator/ceph/nvmeof/spec_test.go b/pkg/operator/ceph/nvmeof/spec_test.go index 3cef9141e..2ac32e17d 100644 --- a/pkg/operator/ceph/nvmeof/spec_test.go +++ b/pkg/operator/ceph/nvmeof/spec_test.go @@ -187,6 +187,95 @@ func TestDeploymentSpec(t *testing.T) { assert.GreaterOrEqual(t, nvmeofCont.LivenessProbe.TimeoutSeconds, int32(5)) }) + t.Run("applies default placement when not specified", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 3, + }, + } + + r, configHash := newDeploymentSpecTest(t) + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + + if assert.NotNil(t, d.Spec.Template.Spec.Affinity) { + assert.Nil(t, d.Spec.Template.Spec.Affinity.PodAntiAffinity) + } + + assert.Len(t, d.Spec.Template.Spec.TopologySpreadConstraints, 1) + assert.Equal(t, int32(1), d.Spec.Template.Spec.TopologySpreadConstraints[0].MaxSkew) + assert.Equal(t, "kubernetes.io/hostname", d.Spec.Template.Spec.TopologySpreadConstraints[0].TopologyKey) + assert.Equal(t, v1.ScheduleAnyway, d.Spec.Template.Spec.TopologySpreadConstraints[0].WhenUnsatisfiable) + assert.Equal(t, map[string]string{"app": AppName}, d.Spec.Template.Spec.TopologySpreadConstraints[0].LabelSelector.MatchLabels) + }) + + t.Run("user placement overrides default placement", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 3, + Placement: cephv1.Placement{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + TopologyKey: "topology.kubernetes.io/zone", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "custom-app", + }, + }, + }, + }, + }, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + { + MaxSkew: 2, + TopologyKey: "topology.kubernetes.io/zone", + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "custom-app", + }, + }, + }, + }, + }, + }, + } + + r, configHash := newDeploymentSpecTest(t) + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + + assert.NotNil(t, d.Spec.Template.Spec.Affinity) + assert.NotNil(t, d.Spec.Template.Spec.Affinity.PodAntiAffinity) + assert.Len(t, d.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 1) + assert.Empty(t, d.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) + assert.Equal(t, "topology.kubernetes.io/zone", d.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey) + assert.Equal(t, map[string]string{"app": "custom-app"}, d.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels) + + assert.Len(t, d.Spec.Template.Spec.TopologySpreadConstraints, 1) + assert.Equal(t, int32(2), d.Spec.Template.Spec.TopologySpreadConstraints[0].MaxSkew) + assert.Equal(t, "topology.kubernetes.io/zone", d.Spec.Template.Spec.TopologySpreadConstraints[0].TopologyKey) + assert.Equal(t, v1.DoNotSchedule, d.Spec.Template.Spec.TopologySpreadConstraints[0].WhenUnsatisfiable) + assert.Equal(t, map[string]string{"app": "custom-app"}, d.Spec.Template.Spec.TopologySpreadConstraints[0].LabelSelector.MatchLabels) + }) + t.Run("with host network", func(t *testing.T) { nvmeof := &cephv1.CephNVMeOFGateway{ ObjectMeta: metav1.ObjectMeta{