diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index d8bb2b3ce..3d6c76a9c 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -4537,8 +4537,11 @@ string +(Optional)

Image is the container image to use for the NVMe-oF gateway daemon. -For example, quay.io/ceph/nvmeof:1.5

+For example, quay.io/ceph/nvmeof:1.5 +If not specified, the image is fetched from the Ceph mon config store +(mgr/cephadm/container_image_nvmeof).

@@ -10803,8 +10806,11 @@ string +(Optional)

Image is the container image to use for the NVMe-oF gateway daemon. -For example, quay.io/ceph/nvmeof:1.5

+For example, quay.io/ceph/nvmeof:1.5 +If not specified, the image is fetched from the Ceph mon config store +(mgr/cephadm/container_image_nvmeof).

diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index bec02f5d1..4fa1bfd1e 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -11703,6 +11703,9 @@ spec: description: |- Image is the container image to use for the NVMe-oF gateway daemon. For example, quay.io/ceph/nvmeof:1.5 + If not specified, the image is fetched from the Ceph mon config store + (mgr/cephadm/container_image_nvmeof). + maxLength: 1024 minLength: 1 type: string instances: @@ -12475,7 +12478,6 @@ spec: type: object required: - group - - image - instances - pool type: object diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 270f062bc..5bf98f031 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -11695,6 +11695,9 @@ spec: description: |- Image is the container image to use for the NVMe-oF gateway daemon. For example, quay.io/ceph/nvmeof:1.5 + If not specified, the image is fetched from the Ceph mon config store + (mgr/cephadm/container_image_nvmeof). + maxLength: 1024 minLength: 1 type: string instances: @@ -12467,7 +12470,6 @@ spec: type: object required: - group - - image - instances - pool type: object diff --git a/deploy/examples/nvmeof-test.yaml b/deploy/examples/nvmeof-test.yaml index 58a4e1c95..98a332fca 100644 --- a/deploy/examples/nvmeof-test.yaml +++ b/deploy/examples/nvmeof-test.yaml @@ -6,7 +6,10 @@ metadata: name: nvmeof namespace: rook-ceph # namespace:cluster spec: - image: quay.io/ceph/nvmeof:1.5 + # image is optional. If not specified, the image is fetched from the Ceph config store + # (ceph config get mgr mgr/cephadm/container_image_nvmeof). + # Set the image property to override the default: + # image: quay.io/ceph/nvmeof: pool: nvmeof group: group-a instances: 1 diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 9b5bd5424..09cdffe93 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -4212,8 +4212,12 @@ type CephNVMeOFGatewayList struct { type NVMeOFGatewaySpec struct { // Image is the container image to use for the NVMe-oF gateway daemon. // For example, quay.io/ceph/nvmeof:1.5 + // If not specified, the image is fetched from the Ceph mon config store + // (mgr/cephadm/container_image_nvmeof). + // +optional // +kubebuilder:validation:MinLength=1 - Image string `json:"image"` + // +kubebuilder:validation:MaxLength=1024 + Image string `json:"image,omitempty"` // The number of active gateway instances // +kubebuilder:validation:Minimum=1 diff --git a/pkg/operator/ceph/nvmeof/controller_test.go b/pkg/operator/ceph/nvmeof/controller_test.go index e398667bb..281fcdbf3 100644 --- a/pkg/operator/ceph/nvmeof/controller_test.go +++ b/pkg/operator/ceph/nvmeof/controller_test.go @@ -92,6 +92,13 @@ func TestCephNVMeOFGatewayController(t *testing.T) { } panic(fmt.Sprintf("unhandled command %s %v", command, args)) }, + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) { + logger.Infof("mock execute with timeout: %s %v", command, arg) + if command == "ceph" && arg[0] == "config" && arg[1] == "get" { + return "quay.io/ceph/nvmeof:1.5", nil + } + return "", nil + }, } } @@ -505,6 +512,12 @@ func TestNVMeOFKeyRotation(t *testing.T) { } return "", nil }, + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) { + if command == "ceph" && arg[0] == "config" && arg[1] == "get" { + return "quay.io/ceph/nvmeof:1.5", nil + } + return "", nil + }, } clientset := test.New(t, 3) c := &clusterd.Context{ diff --git a/pkg/operator/ceph/nvmeof/spec.go b/pkg/operator/ceph/nvmeof/spec.go index a88819c08..5f73904b0 100644 --- a/pkg/operator/ceph/nvmeof/spec.go +++ b/pkg/operator/ceph/nvmeof/spec.go @@ -41,7 +41,6 @@ const ( nvmeofGatewayPort = 5500 nvmeofMonitorPort = 5499 nvmeofDiscoveryPort = 8009 - defaultNVMeOFImage = "quay.io/ceph/nvmeof:1.5" configKey = "config" serviceAccountName = "rook-ceph-nvmeof" ) @@ -182,7 +181,10 @@ func (r *ReconcileCephNVMeOFGateway) makeDeployment(nvmeof *cephv1.CephNVMeOFGat gatewayConfigVol, gatewayConfigMount := gatewayConfigVolumeAndMount(configMapName) initContainer := r.createCephConfigInitContainer(nvmeof, daemonID, gatewayConfigMount) - daemonContainer := r.daemonContainer(nvmeof, cephConfigMount) + daemonContainer, err := r.daemonContainer(nvmeof, cephConfigMount) + if err != nil { + return nil, err + } gatewayName := instanceName(nvmeof, daemonID) podSpec := v1.PodSpec{ @@ -352,7 +354,12 @@ func (r *ReconcileCephNVMeOFGateway) createCephConfigInitContainer(nvmeof *cephv return container } -func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGateway, cephConfigMount v1.VolumeMount) v1.Container { +func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGateway, cephConfigMount v1.VolumeMount) (v1.Container, error) { + image, err := r.getNVMeOFImage(nvmeof) + if err != nil { + return v1.Container{}, err + } + privileged := true container := v1.Container{ Name: "nvmeof-gateway", @@ -361,7 +368,7 @@ func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGa "/etc/ceph/nvmeof.conf", }, - Image: getNVMeOFImage(nvmeof), + Image: image, ImagePullPolicy: v1.PullIfNotPresent, VolumeMounts: []v1.VolumeMount{ cephConfigMount, @@ -423,7 +430,7 @@ func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGa container.LivenessProbe = r.defaultLivenessProbe(nvmeof) } result := cephconfig.ConfigureLivenessProbe(container, nvmeof.Spec.LivenessProbe) - return result + return result, nil } func (r *ReconcileCephNVMeOFGateway) defaultLivenessProbe(nvmeof *cephv1.CephNVMeOFGateway) *v1.Probe { @@ -443,12 +450,25 @@ func instanceName(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) string { } // getNVMeOFImage returns the image to use for the NVMe-oF gateway. -// If the image is specified in the spec, it is used; otherwise, the default image is returned. -func getNVMeOFImage(nvmeof *cephv1.CephNVMeOFGateway) string { +// Priority: CR spec.image > ceph config (mgr/cephadm/container_image_nvmeof). +// If neither source provides an image, an error is returned to fail the reconcile. +func (r *ReconcileCephNVMeOFGateway) getNVMeOFImage(nvmeof *cephv1.CephNVMeOFGateway) (string, error) { if nvmeof.Spec.Image != "" { - return nvmeof.Spec.Image + logger.Infof("using NVMe-oF gateway image from CR spec: %s", nvmeof.Spec.Image) + return nvmeof.Spec.Image, nil } - return defaultNVMeOFImage + + monStore := cephconfig.GetMonStore(r.context, r.clusterInfo) + image, err := monStore.Get("mgr", "mgr/cephadm/container_image_nvmeof") + if err != nil { + return "", errors.Wrap(err, "failed to get NVMe-oF image from ceph config (mgr/cephadm/container_image_nvmeof)") + } + if image == "" { + return "", errors.New("NVMe-oF gateway image not specified in the CR and ceph config key mgr/cephadm/container_image_nvmeof is empty") + } + + logger.Infof("using NVMe-oF gateway image from ceph config: %s", image) + return image, nil } func gatewayConfigVolumeAndMount(configConfigMap string) (v1.Volume, v1.VolumeMount) { diff --git a/pkg/operator/ceph/nvmeof/spec_test.go b/pkg/operator/ceph/nvmeof/spec_test.go index 2ac32e17d..7ebb06c09 100644 --- a/pkg/operator/ceph/nvmeof/spec_test.go +++ b/pkg/operator/ceph/nvmeof/spec_test.go @@ -17,9 +17,11 @@ limitations under the License. package nvmeof import ( + "context" "fmt" "strings" "testing" + "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" @@ -72,6 +74,7 @@ func newDeploymentSpecTest(t *testing.T) (*ReconcileCephNVMeOFGateway, string) { clusterInfo: &cephclient.ClusterInfo{ FSID: "myfsid", CephVersion: cephver.Squid, + Context: context.TODO(), }, cephClusterSpec: &cephv1.ClusterSpec{ CephVersion: cephv1.CephVersionSpec{ @@ -423,14 +426,13 @@ func TestDeploymentSpec(t *testing.T) { assert.True(t, *daemonCont.SecurityContext.Privileged) }) - t.Run("daemon container uses default image when not specified", func(t *testing.T) { + t.Run("daemon container uses image from ceph config when not in CR", func(t *testing.T) { nvmeof := &cephv1.CephNVMeOFGateway{ ObjectMeta: metav1.ObjectMeta{ Name: "my-nvmeof", Namespace: "rook-ceph-test-ns", }, Spec: cephv1.NVMeOFGatewaySpec{ - // Image not specified - should use default Pool: "nvmeof", Group: "group-a", Instances: 1, @@ -438,6 +440,12 @@ func TestDeploymentSpec(t *testing.T) { } r, configHash := newDeploymentSpecTest(t) + executor := &exectest.MockExecutor{ + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) { + return "quay.io/ceph/nvmeof:1.5", nil + }, + } + r.context.Executor = executor configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) @@ -449,6 +457,33 @@ func TestDeploymentSpec(t *testing.T) { assert.Equal(t, "quay.io/ceph/nvmeof:1.5", daemonCont.Image) }) + t.Run("daemon container fails when image not in CR and ceph config is empty", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + } + + r, configHash := newDeploymentSpecTest(t) + executor := &exectest.MockExecutor{ + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) { + return "", nil + }, + } + r.context.Executor = executor + + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + _, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.Error(t, err) + assert.Contains(t, err.Error(), "NVMe-oF gateway image not specified") + }) + t.Run("hostname set when instance name is valid", func(t *testing.T) { nvmeof := &cephv1.CephNVMeOFGateway{ ObjectMeta: metav1.ObjectMeta{