nvmeof: fetch gateway image from ceph config when not in CR
Make the NVMe-oF gateway image optional in the CR spec.
When not specified, the operator fetches the image from
the Ceph mon config store using the key
mgr/cephadm/container_image_nvmeof. Falls back to the
hardcoded default if the config is unavailable.
Signed-off-by: Oded Viner <oviner@redhat.com>
(cherry picked from commit 8b4fd67e17)
This commit is contained in:
1 parent
241ba19fa4
commit
2a2dc44cb7
8 files changed
+102
-17
No files matched your search
@@ -4537,8 +4537,11 @@ string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
For example, quay.io/ceph/nvmeof:1.5</p>
|
||||
For example, quay.io/ceph/nvmeof:1.5
|
||||
If not specified, the image is fetched from the Ceph mon config store
|
||||
(mgr/cephadm/container_image_nvmeof).</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -10803,8 +10806,11 @@ string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
For example, quay.io/ceph/nvmeof:1.5</p>
|
||||
For example, quay.io/ceph/nvmeof:1.5
|
||||
If not specified, the image is fetched from the Ceph mon config store
|
||||
(mgr/cephadm/container_image_nvmeof).</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
||||
@@ -11703,6 +11703,9 @@ spec:
|
||||
description: |-
|
||||
Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
For example, quay.io/ceph/nvmeof:1.5
|
||||
If not specified, the image is fetched from the Ceph mon config store
|
||||
(mgr/cephadm/container_image_nvmeof).
|
||||
maxLength: 1024
|
||||
minLength: 1
|
||||
type: string
|
||||
instances:
|
||||
@@ -12475,7 +12478,6 @@ spec:
|
||||
type: object
|
||||
required:
|
||||
- group
|
||||
- image
|
||||
- instances
|
||||
- pool
|
||||
type: object
|
||||
|
||||
@@ -11695,6 +11695,9 @@ spec:
|
||||
description: |-
|
||||
Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
For example, quay.io/ceph/nvmeof:1.5
|
||||
If not specified, the image is fetched from the Ceph mon config store
|
||||
(mgr/cephadm/container_image_nvmeof).
|
||||
maxLength: 1024
|
||||
minLength: 1
|
||||
type: string
|
||||
instances:
|
||||
@@ -12467,7 +12470,6 @@ spec:
|
||||
type: object
|
||||
required:
|
||||
- group
|
||||
- image
|
||||
- instances
|
||||
- pool
|
||||
type: object
|
||||
|
||||
@@ -6,7 +6,10 @@ metadata:
|
||||
name: nvmeof
|
||||
namespace: rook-ceph # namespace:cluster
|
||||
spec:
|
||||
image: quay.io/ceph/nvmeof:1.5
|
||||
# image is optional. If not specified, the image is fetched from the Ceph config store
|
||||
# (ceph config get mgr mgr/cephadm/container_image_nvmeof).
|
||||
# Set the image property to override the default:
|
||||
# image: quay.io/ceph/nvmeof:<version>
|
||||
pool: nvmeof
|
||||
group: group-a
|
||||
instances: 1
|
||||
|
||||
@@ -4212,8 +4212,12 @@ type CephNVMeOFGatewayList struct {
|
||||
type NVMeOFGatewaySpec struct {
|
||||
// Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
// For example, quay.io/ceph/nvmeof:1.5
|
||||
// If not specified, the image is fetched from the Ceph mon config store
|
||||
// (mgr/cephadm/container_image_nvmeof).
|
||||
// +optional
|
||||
// +kubebuilder:validation:MinLength=1
|
||||
Image string `json:"image"`
|
||||
// +kubebuilder:validation:MaxLength=1024
|
||||
Image string `json:"image,omitempty"`
|
||||
|
||||
// The number of active gateway instances
|
||||
// +kubebuilder:validation:Minimum=1
|
||||
|
||||
@@ -92,6 +92,13 @@ func TestCephNVMeOFGatewayController(t *testing.T) {
|
||||
}
|
||||
panic(fmt.Sprintf("unhandled command %s %v", command, args))
|
||||
},
|
||||
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) {
|
||||
logger.Infof("mock execute with timeout: %s %v", command, arg)
|
||||
if command == "ceph" && arg[0] == "config" && arg[1] == "get" {
|
||||
return "quay.io/ceph/nvmeof:1.5", nil
|
||||
}
|
||||
return "", nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -505,6 +512,12 @@ func TestNVMeOFKeyRotation(t *testing.T) {
|
||||
}
|
||||
return "", nil
|
||||
},
|
||||
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) {
|
||||
if command == "ceph" && arg[0] == "config" && arg[1] == "get" {
|
||||
return "quay.io/ceph/nvmeof:1.5", nil
|
||||
}
|
||||
return "", nil
|
||||
},
|
||||
}
|
||||
clientset := test.New(t, 3)
|
||||
c := &clusterd.Context{
|
||||
|
||||
@@ -41,7 +41,6 @@ const (
|
||||
nvmeofGatewayPort = 5500
|
||||
nvmeofMonitorPort = 5499
|
||||
nvmeofDiscoveryPort = 8009
|
||||
defaultNVMeOFImage = "quay.io/ceph/nvmeof:1.5"
|
||||
configKey = "config"
|
||||
serviceAccountName = "rook-ceph-nvmeof"
|
||||
)
|
||||
@@ -182,7 +181,10 @@ func (r *ReconcileCephNVMeOFGateway) makeDeployment(nvmeof *cephv1.CephNVMeOFGat
|
||||
gatewayConfigVol, gatewayConfigMount := gatewayConfigVolumeAndMount(configMapName)
|
||||
|
||||
initContainer := r.createCephConfigInitContainer(nvmeof, daemonID, gatewayConfigMount)
|
||||
daemonContainer := r.daemonContainer(nvmeof, cephConfigMount)
|
||||
daemonContainer, err := r.daemonContainer(nvmeof, cephConfigMount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
gatewayName := instanceName(nvmeof, daemonID)
|
||||
podSpec := v1.PodSpec{
|
||||
@@ -352,7 +354,12 @@ func (r *ReconcileCephNVMeOFGateway) createCephConfigInitContainer(nvmeof *cephv
|
||||
return container
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGateway, cephConfigMount v1.VolumeMount) v1.Container {
|
||||
func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGateway, cephConfigMount v1.VolumeMount) (v1.Container, error) {
|
||||
image, err := r.getNVMeOFImage(nvmeof)
|
||||
if err != nil {
|
||||
return v1.Container{}, err
|
||||
}
|
||||
|
||||
privileged := true
|
||||
container := v1.Container{
|
||||
Name: "nvmeof-gateway",
|
||||
@@ -361,7 +368,7 @@ func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGa
|
||||
"/etc/ceph/nvmeof.conf",
|
||||
},
|
||||
|
||||
Image: getNVMeOFImage(nvmeof),
|
||||
Image: image,
|
||||
ImagePullPolicy: v1.PullIfNotPresent,
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
cephConfigMount,
|
||||
@@ -423,7 +430,7 @@ func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGa
|
||||
container.LivenessProbe = r.defaultLivenessProbe(nvmeof)
|
||||
}
|
||||
result := cephconfig.ConfigureLivenessProbe(container, nvmeof.Spec.LivenessProbe)
|
||||
return result
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) defaultLivenessProbe(nvmeof *cephv1.CephNVMeOFGateway) *v1.Probe {
|
||||
@@ -443,12 +450,25 @@ func instanceName(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) string {
|
||||
}
|
||||
|
||||
// getNVMeOFImage returns the image to use for the NVMe-oF gateway.
|
||||
// If the image is specified in the spec, it is used; otherwise, the default image is returned.
|
||||
func getNVMeOFImage(nvmeof *cephv1.CephNVMeOFGateway) string {
|
||||
// Priority: CR spec.image > ceph config (mgr/cephadm/container_image_nvmeof).
|
||||
// If neither source provides an image, an error is returned to fail the reconcile.
|
||||
func (r *ReconcileCephNVMeOFGateway) getNVMeOFImage(nvmeof *cephv1.CephNVMeOFGateway) (string, error) {
|
||||
if nvmeof.Spec.Image != "" {
|
||||
return nvmeof.Spec.Image
|
||||
logger.Infof("using NVMe-oF gateway image from CR spec: %s", nvmeof.Spec.Image)
|
||||
return nvmeof.Spec.Image, nil
|
||||
}
|
||||
return defaultNVMeOFImage
|
||||
|
||||
monStore := cephconfig.GetMonStore(r.context, r.clusterInfo)
|
||||
image, err := monStore.Get("mgr", "mgr/cephadm/container_image_nvmeof")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to get NVMe-oF image from ceph config (mgr/cephadm/container_image_nvmeof)")
|
||||
}
|
||||
if image == "" {
|
||||
return "", errors.New("NVMe-oF gateway image not specified in the CR and ceph config key mgr/cephadm/container_image_nvmeof is empty")
|
||||
}
|
||||
|
||||
logger.Infof("using NVMe-oF gateway image from ceph config: %s", image)
|
||||
return image, nil
|
||||
}
|
||||
|
||||
func gatewayConfigVolumeAndMount(configConfigMap string) (v1.Volume, v1.VolumeMount) {
|
||||
|
||||
@@ -17,9 +17,11 @@ limitations under the License.
|
||||
package nvmeof
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
|
||||
@@ -72,6 +74,7 @@ func newDeploymentSpecTest(t *testing.T) (*ReconcileCephNVMeOFGateway, string) {
|
||||
clusterInfo: &cephclient.ClusterInfo{
|
||||
FSID: "myfsid",
|
||||
CephVersion: cephver.Squid,
|
||||
Context: context.TODO(),
|
||||
},
|
||||
cephClusterSpec: &cephv1.ClusterSpec{
|
||||
CephVersion: cephv1.CephVersionSpec{
|
||||
@@ -423,14 +426,13 @@ func TestDeploymentSpec(t *testing.T) {
|
||||
assert.True(t, *daemonCont.SecurityContext.Privileged)
|
||||
})
|
||||
|
||||
t.Run("daemon container uses default image when not specified", func(t *testing.T) {
|
||||
t.Run("daemon container uses image from ceph config when not in CR", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
// Image not specified - should use default
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
@@ -438,6 +440,12 @@ func TestDeploymentSpec(t *testing.T) {
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) {
|
||||
return "quay.io/ceph/nvmeof:1.5", nil
|
||||
},
|
||||
}
|
||||
r.context.Executor = executor
|
||||
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
@@ -449,6 +457,33 @@ func TestDeploymentSpec(t *testing.T) {
|
||||
assert.Equal(t, "quay.io/ceph/nvmeof:1.5", daemonCont.Image)
|
||||
})
|
||||
|
||||
t.Run("daemon container fails when image not in CR and ceph config is empty", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) {
|
||||
return "", nil
|
||||
},
|
||||
}
|
||||
r.context.Executor = executor
|
||||
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
_, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "NVMe-oF gateway image not specified")
|
||||
})
|
||||
|
||||
t.Run("hostname set when instance name is valid", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
|
||||
Reference in new issue
Block a user