diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index ed5de28c1..f52965fee 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -8047,6 +8047,39 @@ ProbeSpec If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.

+ + +image
+ +string + + + +(Optional) +

Image is the container image used to launch the Ceph NFS (Ganesha) daemon(s). +The image must include the NFS Ganesha binaries, such as are included with the official Ceph releases. E.g.: quay.io/ceph/ceph: +If not specified, the Ceph image defined in the CephCluster is used. +Overriding the CephCluster defined image is not normally necessary when using the official Ceph images. +The image must contain the NFS Ganesha and dbus packages. +If the SSSD sidecar is enabled, the image must also contain the sssd-client package.

+ + + + +imagePullPolicy
+ + +Kubernetes core/v1.PullPolicy + + + + +(Optional) +

ImagePullPolicy describes a policy for if/when to pull a container image +One of Always, Never, IfNotPresent. +This field only has effect if an image is specified.

+ +

GatewaySpec diff --git a/Documentation/Storage-Configuration/NFS/nfs-advanced.md b/Documentation/Storage-Configuration/NFS/nfs-advanced.md index e26c8d2f2..ba7f72c75 100644 --- a/Documentation/Storage-Configuration/NFS/nfs-advanced.md +++ b/Documentation/Storage-Configuration/NFS/nfs-advanced.md @@ -49,3 +49,31 @@ can be used. This creates an export for the `/testrgw` pseudo path on an existin ```console ceph nfs export create rgw my-nfs /testrgw bkt4exp ``` + +## Deploying a different NFS Ganesha version + +The default behavior is to use `CephCluster.spec.cephVersion.image` as the +basis for the NFS Ganesha pods. This may be overridden in order to change the +NFS Ganesha version independently of the version bundled in the image used for +the other Ceph cluster components. A example of why this might be desirable is +to allow the use of unmodified official Ceph OCI images for all other Ceph +cluster components, while still allowing the version of NFS Ganesha to be +upgraded independently. + +The image must contain the NFS Ganesha and dbus packages. If the SSSD sidecar +is enabled, the image must also contain the sssd-client package. + +```yaml +apiVersion: ceph.rook.io/v1 +kind: CephNFS +metadata: + name: my-nfs + namespace: rook-ceph +spec: + rados: + pool: my-nfs-data0 + server: + active: 1 + image: custom/ganesha-image:latest" + imagePullPolicy: Always +``` diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 0c8f9b0ed..65b7e61ee 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -10719,6 +10719,28 @@ spec: description: Whether host networking is enabled for the Ganesha server. If not set, the network settings from the cluster CR will be applied. nullable: true type: boolean + image: + description: |- + Image is the container image used to launch the Ceph NFS (Ganesha) daemon(s). + The image must include the NFS Ganesha binaries, such as are included with the official Ceph releases. E.g.: quay.io/ceph/ceph: + If not specified, the Ceph image defined in the CephCluster is used. + Overriding the CephCluster defined image is not normally necessary when using the official Ceph images. + The image must contain the NFS Ganesha and dbus packages. + If the SSSD sidecar is enabled, the image must also contain the sssd-client package. + maxLength: 1572864 + minLength: 1 + type: string + imagePullPolicy: + description: |- + ImagePullPolicy describes a policy for if/when to pull a container image + One of Always, Never, IfNotPresent. + This field only has effect if an image is specified. + enum: + - IfNotPresent + - Always + - Never + - "" + type: string labels: additionalProperties: type: string diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 913d7b4c2..87c9f5d76 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -10712,6 +10712,28 @@ spec: description: Whether host networking is enabled for the Ganesha server. If not set, the network settings from the cluster CR will be applied. nullable: true type: boolean + image: + description: |- + Image is the container image used to launch the Ceph NFS (Ganesha) daemon(s). + The image must include the NFS Ganesha binaries, such as are included with the official Ceph releases. E.g.: quay.io/ceph/ceph: + If not specified, the Ceph image defined in the CephCluster is used. + Overriding the CephCluster defined image is not normally necessary when using the official Ceph images. + The image must contain the NFS Ganesha and dbus packages. + If the SSSD sidecar is enabled, the image must also contain the sssd-client package. + maxLength: 1572864 + minLength: 1 + type: string + imagePullPolicy: + description: |- + ImagePullPolicy describes a policy for if/when to pull a container image + One of Always, Never, IfNotPresent. + This field only has effect if an image is specified. + enum: + - IfNotPresent + - Always + - Never + - "" + type: string labels: additionalProperties: type: string diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index aa5b267af..8c12dcbe6 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -2827,6 +2827,24 @@ type GaneshaServerSpec struct { // If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. // +optional LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"` + + // Image is the container image used to launch the Ceph NFS (Ganesha) daemon(s). + // The image must include the NFS Ganesha binaries, such as are included with the official Ceph releases. E.g.: quay.io/ceph/ceph: + // If not specified, the Ceph image defined in the CephCluster is used. + // Overriding the CephCluster defined image is not normally necessary when using the official Ceph images. + // The image must contain the NFS Ganesha and dbus packages. + // If the SSSD sidecar is enabled, the image must also contain the sssd-client package. + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=1572864 + Image string `json:"image,omitempty"` + + // ImagePullPolicy describes a policy for if/when to pull a container image + // One of Always, Never, IfNotPresent. + // This field only has effect if an image is specified. + // +optional + // +kubebuilder:validation:Enum=IfNotPresent;Always;Never;"" + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` } // NFSSecuritySpec represents security configurations for an NFS server pod diff --git a/pkg/operator/ceph/nfs/controller_test.go b/pkg/operator/ceph/nfs/controller_test.go index 923ec845a..24b8da9c5 100644 --- a/pkg/operator/ceph/nfs/controller_test.go +++ b/pkg/operator/ceph/nfs/controller_test.go @@ -36,6 +36,7 @@ import ( "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -496,6 +497,27 @@ func TestCephNFSController(t *testing.T) { assert.Equal(t, "rook-ceph-nfs-my-nfs-a", (*deploymentsUpdated)[0].Name) }) }) + + t.Run("override ganesha OCI image", func(t *testing.T) { + cephNFS := baseCephNFS() + cephNFS.Spec.Server.Image = "custom/ganesha-image:latest" + cephNFS.Spec.Server.ImagePullPolicy = "Always" + + cCtx := newContext(successExecutor(t)) + cl := newControllerClient(cephNFS, cephClusterReady(cCtx)) + r := newReconcile(cCtx, cl) + + res, err := r.Reconcile(ctx, req) + require.NoError(t, err) + assert.Zero(t, res.RequeueAfter) + assertCephNFSReady(t, r) + + deploy, err := cCtx.Clientset.AppsV1().Deployments(namespace).Get(ctx, "rook-ceph-nfs-my-nfs-a", metav1.GetOptions{}) + require.NoError(t, err) + + assert.Equal(t, cephNFS.Spec.Server.Image, deploy.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, cephNFS.Spec.Server.ImagePullPolicy, deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy) + }) } func TestNFSKeyRotation(t *testing.T) { diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index d99277a06..c907a5b41 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -208,6 +208,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig) func (r *ReconcileCephNFS) connectionConfigInitContainer(nfs *cephv1.CephNFS, name string) v1.Container { _, cephConfigMount := cephConfigVolumeAndMount() + // Use the cluster Ceph image to setup keyring even if the Ganesha image has been overridden. return controller.GenerateMinimalCephConfInitContainer( getNFSClientID(nfs, name), keyring.VolumeMount().KeyringFilePath(), @@ -242,8 +243,8 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig "-p", ganeshaPid, // PID file location "-N", logLevel, // Change Log level }, - Image: r.cephClusterSpec.CephVersion.Image, - ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), + Image: r.GetNFSImage(nfs), + ImagePullPolicy: r.GetNFSImagePullPolicy(nfs), VolumeMounts: []v1.VolumeMount{ cephConfigMount, keyring.VolumeMount().Resource(instanceName(nfs, cfg.ID)), @@ -286,12 +287,12 @@ func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container { "--nopidfile", // don't write a pid file // some dbus-daemon versions have flag --nosyslog to send logs to sterr; not ceph upstream image }, - Image: r.cephClusterSpec.CephVersion.Image, - ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), + Image: r.GetNFSImage(nfs), + ImagePullPolicy: r.GetNFSImagePullPolicy(nfs), VolumeMounts: []v1.VolumeMount{ dbusMount, }, - Env: k8sutil.ClusterDaemonEnvVars(r.cephClusterSpec.CephVersion.Image), // do not need access to Ceph env vars b/c not a Ceph daemon + Env: k8sutil.ClusterDaemonEnvVars(r.GetNFSImage(nfs)), // do not need access to Ceph env vars b/c not a Ceph daemon Resources: nfs.Spec.Server.Resources, SecurityContext: &v1.SecurityContext{ RunAsUser: &dbusUID, @@ -299,6 +300,22 @@ func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container { } } +func (r *ReconcileCephNFS) GetNFSImage(nfs *cephv1.CephNFS) string { + if nfs.Spec.Server.Image != "" { + return nfs.Spec.Server.Image + } + + return r.cephClusterSpec.CephVersion.Image +} + +func (r *ReconcileCephNFS) GetNFSImagePullPolicy(nfs *cephv1.CephNFS) v1.PullPolicy { + if nfs.Spec.Server.Image != "" { + return controller.GetContainerImagePullPolicy(nfs.Spec.Server.ImagePullPolicy) + } + + return controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy) +} + func getLabels(n *cephv1.CephNFS, name string, includeNewLabels bool) map[string]string { labels := controller.CephDaemonAppLabels(AppName, n.Namespace, "nfs", n.Name+"-"+name, n.Name, "cephnfses.ceph.rook.io", includeNewLabels) labels[CephNFSNameLabelKey] = n.Name