From 8e1122ecfafa70a2a5bda14cf491fea8a10ff543 Mon Sep 17 00:00:00 2001 From: Oded Viner Date: Sun, 9 Nov 2025 11:38:01 +0200 Subject: [PATCH] nvmeof: add nvme-of gateway crd support adds CephNVMeOFGateway crd for managing ceph nvme-of gateways. supports configurable instances, placement, resources, and networking options. Signed-off-by: Oded Viner (cherry picked from commit 88a9bf5c7f8e62caea75aed4ce580f222ee9f7aa) --- .commitlintrc.json | 1 + Documentation/CRDs/specification.md | 559 ++++++++++- .../Block-Storage-RBD/nvme-of.md | 462 +++++++++ .../library/templates/_cluster-role.tpl | 1 + .../templates/_cluster-serviceaccount.tpl | 10 + .../rook-ceph/templates/clusterrole.yaml | 3 + .../charts/rook-ceph/templates/resources.yaml | 914 ++++++++++++++++++ deploy/examples/common.yaml | 18 + deploy/examples/crds.yaml | 913 +++++++++++++++++ deploy/examples/csi/nvmeof/node-plugin.yaml | 202 ++++ deploy/examples/csi/nvmeof/nvmeof-pool.yaml | 9 + deploy/examples/csi/nvmeof/pod.yaml | 23 + deploy/examples/csi/nvmeof/provisioner.yaml | 336 +++++++ deploy/examples/csi/nvmeof/pvc.yaml | 12 + deploy/examples/csi/nvmeof/storageclass.yaml | 31 + deploy/examples/nvmeof-test.yaml | 13 + deploy/examples/operator-openshift.yaml | 4 +- pkg/apis/ceph.rook.io/v1/nvmeof.go | 26 + pkg/apis/ceph.rook.io/v1/register.go | 2 + pkg/apis/ceph.rook.io/v1/types.go | 122 +++ .../ceph.rook.io/v1/zz_generated.deepcopy.go | 163 ++++ .../ceph.rook.io/v1/ceph.rook.io_client.go | 5 + .../ceph.rook.io/v1/cephnvmeofgateway.go | 68 ++ .../v1/fake/fake_ceph.rook.io_client.go | 4 + .../v1/fake/fake_cephnvmeofgateway.go | 52 + .../ceph.rook.io/v1/generated_expansion.go | 2 + .../ceph.rook.io/v1/cephnvmeofgateway.go | 102 ++ .../ceph.rook.io/v1/interface.go | 7 + .../informers/externalversions/generic.go | 2 + .../ceph.rook.io/v1/cephnvmeofgateway.go | 70 ++ .../ceph.rook.io/v1/expansion_generated.go | 8 + pkg/operator/ceph/cr_manager.go | 2 + pkg/operator/ceph/nvmeof/connectionconfig.sh | 26 + pkg/operator/ceph/nvmeof/controller.go | 587 +++++++++++ pkg/operator/ceph/nvmeof/controller_test.go | 712 ++++++++++++++ pkg/operator/ceph/nvmeof/spec.go | 474 +++++++++ pkg/operator/ceph/nvmeof/spec_test.go | 536 ++++++++++ 37 files changed, 6474 insertions(+), 7 deletions(-) create mode 100644 Documentation/Storage-Configuration/Block-Storage-RBD/nvme-of.md create mode 100644 deploy/examples/csi/nvmeof/node-plugin.yaml create mode 100644 deploy/examples/csi/nvmeof/nvmeof-pool.yaml create mode 100644 deploy/examples/csi/nvmeof/pod.yaml create mode 100644 deploy/examples/csi/nvmeof/provisioner.yaml create mode 100644 deploy/examples/csi/nvmeof/pvc.yaml create mode 100644 deploy/examples/csi/nvmeof/storageclass.yaml create mode 100644 deploy/examples/nvmeof-test.yaml create mode 100644 pkg/apis/ceph.rook.io/v1/nvmeof.go create mode 100644 pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephnvmeofgateway.go create mode 100644 pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephnvmeofgateway.go create mode 100644 pkg/client/informers/externalversions/ceph.rook.io/v1/cephnvmeofgateway.go create mode 100644 pkg/client/listers/ceph.rook.io/v1/cephnvmeofgateway.go create mode 100644 pkg/operator/ceph/nvmeof/connectionconfig.sh create mode 100644 pkg/operator/ceph/nvmeof/controller.go create mode 100644 pkg/operator/ceph/nvmeof/controller_test.go create mode 100644 pkg/operator/ceph/nvmeof/spec.go create mode 100644 pkg/operator/ceph/nvmeof/spec_test.go diff --git a/.commitlintrc.json b/.commitlintrc.json index 95c8309b1..f82329e14 100644 --- a/.commitlintrc.json +++ b/.commitlintrc.json @@ -29,6 +29,7 @@ "monitoring", "multus", "network", + "nvmeof", "nfs", "object", "operator", diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 2478171f4..33be9b2aa 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -2803,7 +2803,7 @@ CIDRList

Annotations (map[string]string alias)

-(Appears on:FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, RBDMirroringSpec, RGWServiceSpec) +(Appears on:FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, NVMeOFGatewaySpec, RBDMirroringSpec, RGWServiceSpec)

Annotations are annotations

@@ -4377,6 +4377,244 @@ string +

CephNVMeOFGateway +

+
+

CephNVMeOFGateway represents a Ceph NVMe-oF Gateway

+
+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+metadata
+ + +Kubernetes meta/v1.ObjectMeta + + +		 +Refer to the Kubernetes API documentation for the fields of the +metadata field. +
+spec
+ + +NVMeOFGatewaySpec + + +		 +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+image
+ +string + +		 +

Image is the container image to use for the NVMe-oF gateway daemon. +For example, quay.io/ceph/nvmeof:1.5

+
+instances
+ +int + +		 +

The number of active gateway instances

+
+pool
+ +string + +		 +

Pool is the RADOS pool where NVMe-oF configuration is stored

+
+group
+ +string + +		 +

Group is the gateway group name for high availability (ANA group)

+
+configMapRef
+ +string + +		 +(Optional) +

ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration +If not specified, a default configuration will be generated

+
+nvmeofConfig
+ +map[string]map[string]string + +		 +(Optional) +

NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration +This allows users to override or add configuration options without needing to manage a ConfigMap

+
+placement
+ + +Placement + + +		 +(Optional) +

The affinity to place the gateway pods

+
+annotations
+ + +Annotations + + +		 +(Optional) +

The annotations-related configuration to add/set on each Pod related object.

+
+labels
+ + +Labels + + +		 +(Optional) +

The labels-related configuration to add/set on each Pod related object.

+
+resources
+ + +Kubernetes core/v1.ResourceRequirements + + +		 +(Optional) +

Resources set resource requests and limits

+
+priorityClassName
+ +string + +		 +(Optional) +

PriorityClassName sets the priority class on the pods

+
+hostNetwork
+ +bool + +		 +(Optional) +

Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.

+
+ports
+ + +NVMeOFGatewayPorts + + +		 +(Optional) +

Ports configuration for the NVMe-oF gateway

+
+livenessProbe
+ + +ProbeSpec + + +		 +(Optional) +

A liveness-probe to verify that gateway has valid run-time state. +If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.

+
+
+status
+ + +NVMeOFGatewayStatus + + +		 +(Optional) +

CephNetworkType (string alias)

@@ -8799,7 +9037,7 @@ int

Labels (map[string]string alias)

-(Appears on:FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, RBDMirroringSpec) +(Appears on:FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, NVMeOFGatewaySpec, RBDMirroringSpec)

Labels are label for a given daemons

@@ -8815,7 +9053,7 @@ int

LocalCephxStatus

-(Appears on:CephFilesystemStatus, FileMirrorStatus, NFSStatus, ObjectStoreStatus, RBDMirrorStatus) +(Appears on:CephFilesystemStatus, FileMirrorStatus, NFSStatus, NVMeOFGatewayStatus, ObjectStoreStatus, RBDMirrorStatus)

@@ -10207,6 +10445,315 @@ LocalCephxStatus +

NVMeOFGatewayPorts +

+

+(Appears on:NVMeOFGatewaySpec) +

+
+

NVMeOFGatewayPorts represents the port configuration for NVMe-oF gateway

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+ioPort
+ +int32 + +		 +(Optional) +

IOPort is the port for NVMe-oF IO traffic (default: 4420)

+
+gatewayPort
+ +int32 + +		 +(Optional) +

GatewayPort is the port for the gateway service (default: 5500)

+
+monitorPort
+ +int32 + +		 +(Optional) +

MonitorPort is the port for the monitor service (default: 5499)

+
+discoveryPort
+ +int32 + +		 +(Optional) +

DiscoveryPort is the port for discovery service (default: 8009)

+
+

NVMeOFGatewaySpec +

+

+(Appears on:CephNVMeOFGateway) +

+
+

NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+image
+ +string + +		 +

Image is the container image to use for the NVMe-oF gateway daemon. +For example, quay.io/ceph/nvmeof:1.5

+
+instances
+ +int + +		 +

The number of active gateway instances

+
+pool
+ +string + +		 +

Pool is the RADOS pool where NVMe-oF configuration is stored

+
+group
+ +string + +		 +

Group is the gateway group name for high availability (ANA group)

+
+configMapRef
+ +string + +		 +(Optional) +

ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration +If not specified, a default configuration will be generated

+
+nvmeofConfig
+ +map[string]map[string]string + +		 +(Optional) +

NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration +This allows users to override or add configuration options without needing to manage a ConfigMap

+
+placement
+ + +Placement + + +		 +(Optional) +

The affinity to place the gateway pods

+
+annotations
+ + +Annotations + + +		 +(Optional) +

The annotations-related configuration to add/set on each Pod related object.

+
+labels
+ + +Labels + + +		 +(Optional) +

The labels-related configuration to add/set on each Pod related object.

+
+resources
+ + +Kubernetes core/v1.ResourceRequirements + + +		 +(Optional) +

Resources set resource requests and limits

+
+priorityClassName
+ +string + +		 +(Optional) +

PriorityClassName sets the priority class on the pods

+
+hostNetwork
+ +bool + +		 +(Optional) +

Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.

+
+ports
+ + +NVMeOFGatewayPorts + + +		 +(Optional) +

Ports configuration for the NVMe-oF gateway

+
+livenessProbe
+ + +ProbeSpec + + +		 +(Optional) +

A liveness-probe to verify that gateway has valid run-time state. +If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.

+
+

NVMeOFGatewayStatus +

+

+(Appears on:CephNVMeOFGateway) +

+
+

NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway

+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+Status
+ + +Status + + +		 +

+(Members of Status are embedded into this type.) +

+
+cephx
+ + +LocalCephxStatus + + +		 +

NamedBlockPoolSpec

@@ -12299,7 +12846,7 @@ string

Placement

-(Appears on:CephCOSIDriverSpec, FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, RBDMirroringSpec, StorageClassDeviceSet) +(Appears on:CephCOSIDriverSpec, FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, NVMeOFGatewaySpec, RBDMirroringSpec, StorageClassDeviceSet)

Placement is the placement for an object

@@ -12722,7 +13269,7 @@ string

ProbeSpec

-(Appears on:GaneshaServerSpec, MetadataServerSpec, ObjectHealthCheckSpec) +(Appears on:GaneshaServerSpec, MetadataServerSpec, NVMeOFGatewaySpec, ObjectHealthCheckSpec)

ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon

@@ -14216,7 +14763,7 @@ int

Status

-(Appears on:CephBucketNotification, CephObjectRealm, CephObjectZone, CephObjectZoneGroup, FileMirrorStatus, NFSStatus, RBDMirrorStatus) +(Appears on:CephBucketNotification, CephObjectRealm, CephObjectZone, CephObjectZoneGroup, FileMirrorStatus, NFSStatus, NVMeOFGatewayStatus, RBDMirrorStatus)

Status represents the status of an object

diff --git a/Documentation/Storage-Configuration/Block-Storage-RBD/nvme-of.md b/Documentation/Storage-Configuration/Block-Storage-RBD/nvme-of.md new file mode 100644 index 000000000..3d4c70780 --- /dev/null +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/nvme-of.md @@ -0,0 +1,462 @@ +--- +title: NVMe-oF Block Storage +--- +# NVMe-oF Block Storage + +**This feature is experimental** + +NVMe over Fabrics (NVMe-oF) allows RBD volumes to be exposed and accessed via the NVMe/TCP protocol. This enables both Kubernetes pods within the cluster and external clients outside the cluster to connect to Ceph block storage using standard NVMe-oF initiators, providing high-performance block storage access over the network. + +## Goals + +The NVMe-oF integration in Rook serves two primary purposes: + +1. **External Client Access**: Rook serves as a backend for external clients outside the cluster, enabling non-Kubernetes workloads to access Ceph block storage through standard NVMe-oF initiators. This allows organizations to leverage their Ceph storage infrastructure for both containerized and traditional workloads. + +2. **In-Cluster Consumption**: Pods inside the Kubernetes cluster can consume storage via the NVMe-oF protocol, providing an alternative to traditional RBD mounts with potential performance benefits for certain workloads. + +Both use cases are supported, allowing you to choose the appropriate access method based on your specific requirements and deployment scenarios. + +For more background and design details, see the [NVMe-oF gateway design doc](https://github.com/rook/rook/blob/master/design/ceph/ceph-nvmeof-gateway.md). +For the Ceph-CSI NVMe-oF design proposal, see the [ceph-csi NVMe-oF proposal](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/nvme-of.md). + +## Prerequisites + +This guide assumes a Rook cluster as explained in the [Quickstart Guide](../../Getting-Started/quickstart.md). + +### Requirements + +- **Ceph Version**: Ceph v20 (Tentacle) or later +- **Disable the Ceph CSI operator**: We are still updating the Ceph CSI operator with NVMe-oF support. Currently, it is required to disable the CSI operator to test NVMe-oF. In operator.yaml, set `ROOK_USE_CSI_OPERATOR: "false"`. + +## Step 1: Create a Ceph Block Pool + +Before creating the NVMe-oF gateway, you need to create a CephBlockPool that will be used by the gateway: + +```yaml +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: nvmeof + namespace: rook-ceph +spec: + failureDomain: host + replicated: + size: 3 +``` + +Create the pool: + +```console +kubectl create -f deploy/examples/csi/nvmeof/nvmeof-pool.yaml +``` + +## Step 2: Create the NVMe-oF Gateway + +The `CephNVMeOFGateway` CRD manages the NVMe-oF gateway infrastructure. The operator will automatically create the following resources: + +- **Service**: One per gateway instance for service discovery +- **Deployment**: One per gateway instance running the NVMe-oF gateway daemon + +Create the gateway: + +```yaml +apiVersion: ceph.rook.io/v1 +kind: CephNVMeOFGateway +metadata: + name: nvmeof + namespace: rook-ceph +spec: + # Container image for the NVMe-oF gateway daemon + image: quay.io/ceph/nvmeof:1.5 + # Pool name that will be used by the NVMe-oF gateway + pool: nvmeof + # ANA (Asymmetric Namespace Access) group name + group: group-a + # Number of gateway instances to run + instances: 1 + hostNetwork: false +``` + +Apply the gateway configuration: + +```console +kubectl create -f deploy/examples/nvmeof-test.yaml +``` + +Verify the gateway is running: + +```console +kubectl get pod -n rook-ceph -l app=rook-ceph-nvmeof +``` + +**Example Output** + +```console +NAME READY STATUS RESTARTS AGE +rook-ceph-nvmeof-nvmeof-a-85844ff6b8-4r8gj 1/1 Running 0 91s +``` + +## Step 4: Deploy the NVMe-oF CSI Driver + +The NVMe-oF CSI driver handles dynamic provisioning of volumes. Deploy the CSI provisioner with the NVMe-oF driver. + +Deploy the NVMe-oF CSI provisioner from the example manifest: + +```console +kubectl create -f deploy/examples/csi/nvmeof/provisioner.yaml +``` + +Verify the CSI provisioner pod is running: + +```console +kubectl get pods -n rook-ceph -l app=csi-nvmeofplugin-provisioner +``` + +**Example Output** + +```console +NAME READY STATUS RESTARTS AGE +csi-nvmeofplugin-provisioner-65b4fbbc8-jjsqj 4/4 Running 0 75s +``` + +## Step 5: Create the StorageClass + +Create a StorageClass that uses the NVMe-oF CSI driver. You'll need to gather the following information from the gateway: + +1. **nvmeofGatewayAddress**: A stable address for the gateway management API +2. **nvmeofGatewayPort**: The gateway port (default: 5500) +3. **listeners**: A JSON array containing listener information for each gateway instance + +Discover the values to use in the StorageClass: + +1. **nvmeofGatewayAddress**: Use the Service `CLUSTER-IP`. + + ```console + kubectl get service -n rook-ceph rook-ceph-nvmeof-nvmeof-a + ``` + + **Example Output** + + ```console + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + rook-ceph-nvmeof-nvmeof-a ClusterIP 10.106.98.71 4420/TCP,5500/TCP,5499/TCP,8009/TCP 24m + ``` + +2. **listeners.address**: Use the gateway pod IP. + + ```console + kubectl get pods -n rook-ceph -l app=rook-ceph-nvmeof -o wide + + ``` + + **Example Output** + + ```console + NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES + rook-ceph-nvmeof-nvmeof-a-5fd6cd4d46-mrbwk 1/1 Running 0 26m 10.244.0.16 minikube + ``` + +3. **listeners.hostname**: Use the gateway deployment name. + + ```console + kubectl get deployments.apps -n rook-ceph -l app=rook-ceph-nvmeof + ``` + + **Example Output** + + ```console + NAME READY UP-TO-DATE AVAILABLE AGE + rook-ceph-nvmeof-nvmeof-a 1/1 1 1 27m + ``` + +Create the StorageClass: + + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ceph-nvmeof +parameters: + clusterID: rook-ceph + pool: nvmeof + subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph + # Management API - talks to gateway to create subsystems/namespaces + nvmeofGatewayAddress: "10.106.98.71" + nvmeofGatewayPort: "5500" + # Data Plane - worker nodes connect here for actual I/O + # List ALL gateway pods for HA and multipath + listeners: | + [ + { + "address": "10.244.0.16", + "port": 4420, + "hostname": "rook-ceph-nvmeof-nvmeof-a" + } + ] + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + imageFormat: "2" + imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff +provisioner: nvmeof.csi.ceph.com +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: false +``` + +Create the StorageClass: + +```console +kubectl create -f deploy/examples/csi/nvmeof/storageclass.yaml +``` + +## Step 6: Create a PersistentVolumeClaim + +Create a PVC using the NVMe-oF storage class: + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nvmeof-external-volume + namespace: default +spec: + storageClassName: ceph-nvmeof + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 128Mi +``` + +!!! note + This PVC is created for CSI driver provisioning. The volume will be accessible via NVMe-oF protocol by both Kubernetes pods within the cluster and external clients outside the cluster using standard NVMe-oF initiators. + +Create the PVC: + +```console +kubectl create -f deploy/examples/csi/nvmeof/pvc.yaml +``` + +Verify the PVC is bound: + +```console +kubectl get pvc nvmeof-external-volume +``` + +**Example Output** + +```console +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +nvmeof-external-volume Bound pvc-b4108580-5cfa-46d3-beff-320088a5bf3c 128Mi RWO ceph-nvmeof 20m +``` + +## Step 7: Deploy the NVMe-oF CSI Node Plugin + +Deploy the NVMe-oF CSI node plugin: + +```console +kubectl create -f deploy/examples/csi/nvmeof/node-plugin.yaml +``` + +Verify the node plugin pod is running: + +```console +kubectl get pods -n rook-ceph -l app=nvmeof.csi.ceph.com-nodeplugin +``` + +**Example Output** + +```console +NAME READY STATUS RESTARTS AGE +nvmeof.csi.ceph.com-nodeplugin-xnm82 2/2 Running 0 31h +``` + +## Step 8: Accessing Volumes via NVMe-oF + +Once the PVC is created and bound, the volume is available via NVMe-oF. The volume can be accessed by both Kubernetes pods within the cluster and external clients outside the cluster. + +### Access from Kubernetes Pods + +Kubernetes pods can consume NVMe-oF volumes by mounting the PVC directly. The CSI driver handles the NVMe-oF connection automatically when the pod mounts the volume. + +Create a sample pod that mounts the PVC: + +```console +kubectl create -f deploy/examples/csi/nvmeof/pod.yaml +``` + +Verify the pod is running: + +```console +kubectl get pods -n default +``` + +**Example Output** + +```console +NAME READY STATUS RESTARTS AGE +nvmeof-test-pod 1/1 Running 0 29h +``` + +### Access from External Clients + +External clients outside the Kubernetes cluster can connect to the gateway using standard NVMe-oF procedures. + +#### Prerequisites for External Clients + +- **NVMe-oF Initiator**: The client must have the `nvme-tcp` kernel module loaded and `nvme-cli` installed +- **Network Access**: The client must be able to reach the gateway service IP and ports + +#### Discover Subsystems + +From the external client, discover available NVMe-oF subsystems: + +```bash +nvme discover -t tcp -a -s 5500 +``` + +Replace `` with the gateway service ClusterIP or an accessible endpoint. + +#### Connect to Subsystem + +Connect to the discovered subsystem: + +```bash +nvme connect -t tcp -n -a -s 5500 +``` + +Replace: + +- `` with the `subsystemNQN` value from your StorageClass (e.g., `nqn.2016-06.io.spdk:cnode1.rook-ceph`) +- `` with the gateway service IP or pod IP + +#### Access the Volume + +Once connected, the NVMe namespace will appear as a block device on the client: + +```bash +lsblk | grep nvme +``` + +The device will typically appear as `/dev/nvmeXnY` where X is the controller number and Y is the namespace ID. + +#### Format and Mount (Optional) + +If you want to format and mount the device: + +```bash +# Format the device +sudo mkfs.ext4 /dev/nvmeXnY + +# Mount the device +sudo mkdir /mnt/nvmeof +sudo mount /dev/nvmeXnY /mnt/nvmeof +``` + +## High Availability + +For production deployments, configure multiple gateway instances for high availability: + +1. **Increase Gateway Instances**: Set `instances: 2` or higher in the `CephNVMeOFGateway` spec +2. **Update StorageClass Listeners**: Add all gateway instance addresses and instance names to the `listeners` array +3. **Load Balancing**: Each gateway instance has its own Service; list all of them to support multipath/HA + +Example with multiple instances: + +```yaml +spec: + instances: 2 + # ... other settings +``` + +Then update the StorageClass `listeners` to include all gateway instances/services: + +```yaml +listeners: | + [ + { + "address": "10.99.212.218", + "port": 4420, + "hostname": "rook-ceph-nvmeof-nvmeof-a" + }, + { + "address": "10.99.212.219", + "port": 4420, + "hostname": "rook-ceph-nvmeof-nvmeof-b" + } + ] +``` + +## Troubleshooting + +### Check Gateway Pod Logs + +```console +kubectl logs -n rook-ceph -l app=rook-ceph-nvmeof --tail=100 +``` + +### Check CSI Provisioner Logs + +```console +kubectl logs -n rook-ceph -l app=csi-nvmeofplugin-provisioner --tail=100 +``` + +### Verify Gateway Service + +```console +kubectl describe service -n rook-ceph rook-ceph-nvmeof-my-nvmeof-0 +``` + +### Check PVC Events + +```console +kubectl describe pvc nvmeof-external-volume +``` + +### Verify Ceph CSI Config + +Ensure the `rook-ceph-csi-config` ConfigMap exists and contains the cluster configuration: + +```console +kubectl get configmap -n rook-ceph rook-ceph-csi-config -o yaml +``` + +## Teardown + +!!! warning + Deleting the PVC will also delete the underlying RBD image and NVMe namespace. Ensure you have backups if needed. + +To clean up all the artifacts created: + +```console +# Delete the test pod +kubectl delete -f deploy/examples/csi/nvmeof/pod.yaml + +# Delete the PVC +kubectl delete pvc nvmeof-external-volume + +# Delete the StorageClass +kubectl delete storageclass ceph-nvmeof + +# Delete the NVMe-oF CSI node plugin +kubectl delete -f deploy/examples/csi/nvmeof/node-plugin.yaml + +# Delete the NVMe-oF CSI provisioner (includes ServiceAccount/RBAC) +kubectl delete -f deploy/examples/csi/nvmeof/provisioner.yaml + +# Delete the NVMe-oF gateway +kubectl delete -n rook-ceph cephnvmeofgateway.ceph.rook.io my-nvmeof + +# Delete the block pool (optional) +kubectl delete -n rook-ceph cephblockpool.ceph.rook.io nvmeof +``` + +## References + +- [Ceph NVMe-oF Documentation](https://docs.ceph.com/en/latest/rbd/nvmeof-overview/) +- [Ceph NVMe-oF Initiator Guide](https://docs.ceph.com/en/latest/rbd/nvmeof-initiator-linux/) +- [Ceph CSI NVMe-oF Support](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/nvme-of.md) diff --git a/deploy/charts/library/templates/_cluster-role.tpl b/deploy/charts/library/templates/_cluster-role.tpl index a95736641..00cf8c250 100644 --- a/deploy/charts/library/templates/_cluster-role.tpl +++ b/deploy/charts/library/templates/_cluster-role.tpl @@ -65,6 +65,7 @@ rules: - cephblockpools - cephfilesystems - cephnfses + - cephnvmeofgateways - cephobjectstores - cephobjectstoreusers - cephobjectrealms diff --git a/deploy/charts/library/templates/_cluster-serviceaccount.tpl b/deploy/charts/library/templates/_cluster-serviceaccount.tpl index 02bd17f87..1b79a46c7 100644 --- a/deploy/charts/library/templates/_cluster-serviceaccount.tpl +++ b/deploy/charts/library/templates/_cluster-serviceaccount.tpl @@ -62,4 +62,14 @@ metadata: labels: {{- include "library.rook-ceph.labels" . | nindent 4 }} {{- include "library.imagePullSecrets" . | nindent 0 }} +--- +# Service account for NVMe-oF gateway +kind: ServiceAccount +apiVersion: v1 +metadata: + name: rook-ceph-nvmeof + namespace: {{ .Release.Namespace }} # namespace:cluster + labels: + {{- include "library.rook-ceph.labels" . | nindent 4 }} +{{- include "library.imagePullSecrets" . | nindent 0 }} {{- end }} diff --git a/deploy/charts/rook-ceph/templates/clusterrole.yaml b/deploy/charts/rook-ceph/templates/clusterrole.yaml index d1bf92fc9..4f1f0f4e0 100644 --- a/deploy/charts/rook-ceph/templates/clusterrole.yaml +++ b/deploy/charts/rook-ceph/templates/clusterrole.yaml @@ -143,6 +143,7 @@ rules: - cephblockpools - cephfilesystems - cephnfses + - cephnvmeofgateways - cephobjectstores - cephobjectstoreusers - cephobjectrealms @@ -169,6 +170,7 @@ rules: - cephblockpools/status - cephfilesystems/status - cephnfses/status + - cephnvmeofgateways/status - cephobjectstores/status - cephobjectstoreusers/status - cephobjectrealms/status @@ -192,6 +194,7 @@ rules: - cephblockpools/finalizers - cephfilesystems/finalizers - cephnfses/finalizers + - cephnvmeofgateways/finalizers - cephobjectstores/finalizers - cephobjectstoreusers/finalizers - cephobjectrealms/finalizers diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index de5f17437..50ab6361d 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -11510,6 +11510,920 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + helm.sh/resource-policy: keep + name: cephnvmeofgateways.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephNVMeOFGateway + listKind: CephNVMeOFGatewayList + plural: cephnvmeofgateways + shortNames: + - nvmeof + singular: cephnvmeofgateway + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephNVMeOFGateway represents a Ceph NVMe-oF Gateway + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway + properties: + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set on each Pod related object. + type: object + configMapRef: + description: |- + ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration + If not specified, a default configuration will be generated + minLength: 1 + type: string + group: + description: Group is the gateway group name for high availability (ANA group) + minLength: 1 + type: string + hostNetwork: + description: Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied. + type: boolean + image: + description: |- + Image is the container image to use for the NVMe-oF gateway daemon. + For example, quay.io/ceph/nvmeof:1.5 + minLength: 1 + type: string + instances: + description: The number of active gateway instances + minimum: 1 + type: integer + labels: + additionalProperties: + type: string + description: The labels-related configuration to add/set on each Pod related object. + type: object + livenessProbe: + description: |- + A liveness-probe to verify that gateway has valid run-time state. + If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + nvmeofConfig: + additionalProperties: + additionalProperties: + type: string + type: object + description: |- + NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration + This allows users to override or add configuration options without needing to manage a ConfigMap + type: object + placement: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + pool: + description: Pool is the RADOS pool where NVMe-oF configuration is stored + minLength: 1 + type: string + ports: + description: Ports configuration for the NVMe-oF gateway + properties: + discoveryPort: + description: 'DiscoveryPort is the port for discovery service (default: 8009)' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + gatewayPort: + description: 'GatewayPort is the port for the gateway service (default: 5500)' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + ioPort: + description: 'IOPort is the port for NVMe-oF IO traffic (default: 4420)' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + monitorPort: + description: 'MonitorPort is the port for the monitor service (default: 5499)' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object + priorityClassName: + description: PriorityClassName sets the priority class on the pods + type: string + resources: + description: Resources set resource requests and limits + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + required: + - group + - image + - instances + - pool + type: object + status: + description: NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway + properties: + cephx: + properties: + daemon: + description: Daemon shows the CephX key status for local Ceph daemons associated with this resources. + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object + type: object + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed by the controller. + format: int64 + type: integer + phase: + type: string + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml index 05b00aefc..e710ad413 100644 --- a/deploy/examples/common.yaml +++ b/deploy/examples/common.yaml @@ -364,6 +364,7 @@ rules: - cephblockpools - cephfilesystems - cephnfses + - cephnvmeofgateways - cephobjectstores - cephobjectstoreusers - cephobjectrealms @@ -390,6 +391,7 @@ rules: - cephblockpools/status - cephfilesystems/status - cephnfses/status + - cephnvmeofgateways/status - cephobjectstores/status - cephobjectstoreusers/status - cephobjectrealms/status @@ -413,6 +415,7 @@ rules: - cephblockpools/finalizers - cephfilesystems/finalizers - cephnfses/finalizers + - cephnvmeofgateways/finalizers - cephobjectstores/finalizers - cephobjectstoreusers/finalizers - cephobjectrealms/finalizers @@ -1002,6 +1005,7 @@ rules: - cephblockpools - cephfilesystems - cephnfses + - cephnvmeofgateways - cephobjectstores - cephobjectstoreusers - cephobjectrealms @@ -1400,6 +1404,20 @@ metadata: # imagePullSecrets: # - name: my-registry-secret --- +# Service account for NVMe-oF gateway +kind: ServiceAccount +apiVersion: v1 +metadata: + name: rook-ceph-nvmeof + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/name: rook-ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- # Service account for Ceph OSDs kind: ServiceAccount apiVersion: v1 diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index d8fb21cb1..e94e7e0e1 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -11503,6 +11503,919 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: cephnvmeofgateways.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephNVMeOFGateway + listKind: CephNVMeOFGatewayList + plural: cephnvmeofgateways + shortNames: + - nvmeof + singular: cephnvmeofgateway + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: CephNVMeOFGateway represents a Ceph NVMe-oF Gateway + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway + properties: + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set on each Pod related object. + type: object + configMapRef: + description: |- + ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration + If not specified, a default configuration will be generated + minLength: 1 + type: string + group: + description: Group is the gateway group name for high availability (ANA group) + minLength: 1 + type: string + hostNetwork: + description: Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied. + type: boolean + image: + description: |- + Image is the container image to use for the NVMe-oF gateway daemon. + For example, quay.io/ceph/nvmeof:1.5 + minLength: 1 + type: string + instances: + description: The number of active gateway instances + minimum: 1 + type: integer + labels: + additionalProperties: + type: string + description: The labels-related configuration to add/set on each Pod related object. + type: object + livenessProbe: + description: |- + A liveness-probe to verify that gateway has valid run-time state. + If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: |- + Probe describes a health check to be performed against a container to determine whether it is + alive or ready to receive traffic. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + type: object + nvmeofConfig: + additionalProperties: + additionalProperties: + type: string + type: object + description: |- + NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration + This allows users to override or add configuration options without needing to manage a ConfigMap + type: object + placement: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + pool: + description: Pool is the RADOS pool where NVMe-oF configuration is stored + minLength: 1 + type: string + ports: + description: Ports configuration for the NVMe-oF gateway + properties: + discoveryPort: + description: 'DiscoveryPort is the port for discovery service (default: 8009)' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + gatewayPort: + description: 'GatewayPort is the port for the gateway service (default: 5500)' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + ioPort: + description: 'IOPort is the port for NVMe-oF IO traffic (default: 4420)' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + monitorPort: + description: 'MonitorPort is the port for the monitor service (default: 5499)' + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object + priorityClassName: + description: PriorityClassName sets the priority class on the pods + type: string + resources: + description: Resources set resource requests and limits + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + required: + - group + - image + - instances + - pool + type: object + status: + description: NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway + properties: + cephx: + properties: + daemon: + description: Daemon shows the CephX key status for local Ceph daemons associated with this resources. + properties: + keyCephVersion: + description: |- + KeyCephVersion reports the Ceph version that created the current generation's keys. This is + same string format as reported by `CephCluster.status.version.version` to allow them to be + compared. E.g., `20.2.0-0`. + For all newly-created resources, this field set to the version of Ceph that created the key. + The special value "Uninitialized" indicates that keys are being created for the first time. + An empty string indicates that the version is unknown, as expected in brownfield deployments. + type: string + keyGeneration: + description: |- + KeyGeneration represents the CephX key generation for the last successful reconcile. + For all newly-created resources, this field is set to `1`. + When keys are rotated due to any rotation policy, the generation is incremented or updated to + the configured policy generation. + Generation `0` indicates that keys existed prior to the implementation of key tracking. + format: int32 + type: integer + type: object + type: object + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed by the controller. + format: int64 + type: integer + phase: + type: string + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.19.0 diff --git a/deploy/examples/csi/nvmeof/node-plugin.yaml b/deploy/examples/csi/nvmeof/node-plugin.yaml new file mode 100644 index 000000000..2319f04d8 --- /dev/null +++ b/deploy/examples/csi/nvmeof/node-plugin.yaml @@ -0,0 +1,202 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvmeof.csi.ceph.com-nodeplugin + namespace: rook-ceph +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: nvmeof.csi.ceph.com-nodeplugin + template: + metadata: + labels: + app: nvmeof.csi.ceph.com-nodeplugin + spec: + containers: + - args: + - --v=5 + - --type=nvmeof + - --nodeserver=true + - --nodeid=$(NODE_ID) + - --drivername=nvmeof.csi.ceph.com + - --endpoint=unix:///csi/csi.sock + - --pidlimit=-1 + - --csi-addons-endpoint=unix:///csi/csi-addons.sock + - --stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/ + - --logtostderr=false + - --alsologtostderr=true + - --log_file=csi-logs/csi-nvmeofplugin.log + env: + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: NODE_ID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: quay.io/cephcsi/cephcsi:v3.16.0 + imagePullPolicy: Always + name: csi-nvmeofplugin + resources: {} + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + drop: + - All + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /run/mount + name: host-run-mount + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /tmp/csi/keys + name: keys-tmp-dir + - mountPath: /csi + name: plugin-dir + # FIXED: Mount path should end with / to mount as directory + - mountPath: /etc/ceph-csi-config/ + name: ceph-csi-config + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugin-mount-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /etc/selinux + name: etc-selinux + readOnly: true + - mountPath: /run/secrets/tokens + name: oidc-token + readOnly: true + - mountPath: /csi-logs + name: logs-dir + # Log rotation mount - commented out since ConfigMap is optional + # - mountPath: /etc/logrotate.d + # name: log-rotate-dir + # readOnly: true + - args: + - "--v=1" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/nvmeof.csi.ceph.com/csi.sock" + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 + imagePullPolicy: IfNotPresent + name: driver-registrar + resources: {} + securityContext: + capabilities: + drop: + - All + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + priorityClassName: system-node-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + # FIXED: Use rook-csi-rbd-plugin-sa since nvmeof uses RBD backend + # This service account already exists and has the necessary permissions + serviceAccount: rook-csi-rbd-plugin-sa + serviceAccountName: rook-csi-rbd-plugin-sa + terminationGracePeriodSeconds: 30 + # tolerations: + # - effect: NoSchedule + # key: node.ocs.openshift.io/storage + # operator: Equal + # value: "true" + volumes: + - hostPath: + path: /dev + type: "" + name: host-dev + - hostPath: + path: /sys + type: "" + name: host-sys + - hostPath: + path: /run/mount + type: "" + name: host-run-mount + - hostPath: + path: /lib/modules + type: "" + name: lib-modules + - emptyDir: + medium: Memory + name: keys-tmp-dir + # FIXED: Use rook-ceph-csi-config instead of ceph-csi-config + # This ConfigMap contains the cluster configuration with monitor endpoints + - configMap: + name: rook-ceph-csi-config + items: + - key: csi-cluster-config-json + path: config.json + name: ceph-csi-config + - hostPath: + path: /var/lib/kubelet/plugins/nvmeof.csi.ceph.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins + type: Directory + name: plugin-mount-dir + - hostPath: + path: /var/lib/kubelet/pods + type: Directory + name: pods-mount-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /etc/selinux + type: "" + name: etc-selinux + - name: oidc-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: ceph-csi-kms + expirationSeconds: 3600 + path: oidc-token + - hostPath: + path: /var/lib/cephcsi/nvmeof.csi.ceph.com-nodeplugin + type: DirectoryOrCreate + name: logs-dir + # Log rotation ConfigMap - optional, can be created if log rotation is needed + # - configMap: + # defaultMode: 420 + # name: nvmeof.csi.ceph.com-logrotate-config + # name: log-rotate-dir + updateStrategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate diff --git a/deploy/examples/csi/nvmeof/nvmeof-pool.yaml b/deploy/examples/csi/nvmeof/nvmeof-pool.yaml new file mode 100644 index 000000000..3db57b6e8 --- /dev/null +++ b/deploy/examples/csi/nvmeof/nvmeof-pool.yaml @@ -0,0 +1,9 @@ +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: nvmeof + namespace: rook-ceph # namespace:cluster +spec: + failureDomain: host + replicated: + size: 3 diff --git a/deploy/examples/csi/nvmeof/pod.yaml b/deploy/examples/csi/nvmeof/pod.yaml new file mode 100644 index 000000000..396c2a9f4 --- /dev/null +++ b/deploy/examples/csi/nvmeof/pod.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nvmeof-test-pod + namespace: default +spec: + containers: + - name: test-container + image: busybox + command: ["/bin/sh"] + args: + [ + "-c", + "while true; do echo 'Testing NVMe-oF mount'; df -h /mnt/nvmeof; sleep 45; done", + ] + volumeMounts: + - name: nvmeof-volume + mountPath: /mnt/nvmeof + volumes: + - name: nvmeof-volume + persistentVolumeClaim: + claimName: nvmeof-external-volume + restartPolicy: Never diff --git a/deploy/examples/csi/nvmeof/provisioner.yaml b/deploy/examples/csi/nvmeof/provisioner.yaml new file mode 100644 index 000000000..225e1cfbb --- /dev/null +++ b/deploy/examples/csi/nvmeof/provisioner.yaml @@ -0,0 +1,336 @@ +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: nvmeof-csi-provisioner + # replace with non-default namespace name + namespace: rook-ceph + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmeof-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch", "create"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["get", "list", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumegroupreplicationcontents"] + verbs: ["get", "list", "watch"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumegroupreplicationclasses"] + verbs: ["get"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattributesclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmeof-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: nvmeof-csi-provisioner + # replace with non-default namespace name + namespace: rook-ceph +roleRef: + kind: ClusterRole + name: nvmeof-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: rook-ceph + name: nvmeof-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmeof-csi-provisioner-role-cfg + # replace with non-default namespace name + namespace: rook-ceph +subjects: + - kind: ServiceAccount + name: nvmeof-csi-provisioner + # replace with non-default namespace name + namespace: rook-ceph +roleRef: + kind: Role + name: nvmeof-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +# --- +# # ssc based on the Ceph-CSI ssc +# # the "users:" contains a reference to the "rook-ceph" namespace +# # +# kind: SecurityContextConstraints +# apiVersion: security.openshift.io/v1 +# metadata: +# name: "ceph-nvmeof-provisioner" +# # To allow running privilegedContainers +# allowPrivilegedContainer: true +# # SYS_ADMIN is needed for rbd to execute rbd map command +# allowedCapabilities: +# - "SYS_ADMIN" +# # Set to false as we write to RootFilesystem inside container +# readOnlyRootFilesystem: false +# requiredDropCapabilities: +# - ALL +# # The type of volumes which are mounted to csi pods +# volumes: +# - configMap +# - projected +# - emptyDir +# runAsUser: +# type: RunAsAny +# seLinuxContext: +# type: RunAsAny +# users: +# # A user needs to be added for each service account. +# - "system:serviceaccount:rook-ceph:nvmeof-csi-provisioner" +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-nvmeofplugin-provisioner + # replace with non-default namespace name + namespace: rook-ceph +spec: + replicas: 1 + selector: + matchLabels: + app: csi-nvmeofplugin-provisioner + template: + metadata: + annotations: + openshift.io/scc: rook-ceph-csi + labels: + app: csi-nvmeofplugin-provisioner + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-nvmeofplugin-provisioner + topologyKey: "kubernetes.io/hostname" + serviceAccountName: nvmeof-csi-provisioner + priorityClassName: system-cluster-critical + containers: + - name: csi-nvmeofplugin + # for stable functionality replace canary with latest release version + # image: quay.io/gdidi/cephcsi:nvmeof + image: quay.io/cephcsi/cephcsi:v3.16.0 + command: ["/usr/local/bin/cephcsi"] + args: + - "--nodeid=$(NODE_ID)" + - "--type=nvmeof" + - "--controllerserver=true" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=nvmeof.csi.ceph.com" + - "--pidlimit=-1" + - "--rbdhardmaxclonedepth=8" + - "--rbdsoftmaxclonedepth=4" + - "--enableprofiling=false" + - "--setmetadata=true" + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + securityContext: + privileged: true + capabilities: + drop: + - ALL + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + - name: oidc-token + mountPath: /run/secrets/tokens + readOnly: true + - name: csi-provisioner + # image: registry.k8s.io/sig-storage/csi-provisioner:v5.1.0 + image: quay.io/nixpanic/csi-provisioner:pr1440 + args: + - "--csi-address=$(ADDRESS)" + - "--v=1" + - "--timeout=150s" + - "--retry-interval-start=500ms" + - "--leader-election=true" + - "--feature-gates=HonorPVReclaimPolicy=true" + - "--prevent-volume-mode-conversion=true" + # if fstype is not specified in storageclass, ext4 is default + - "--default-fstype=ext4" + - "--extra-create-metadata=true" + - "--immediate-topology=false" + - "--http-endpoint=$(POD_IP):8090" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8090 + name: provisioner + protocol: TCP + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.0 + args: + - "--v=1" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--retry-interval-start=500ms" + - "--default-fstype=ext4" + - "--http-endpoint=$(POD_IP):8093" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8093 + name: attacher + protocol: TCP + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-resizer + image: quay.io/nixpanic/csi-resizer:pr544 + # image: registry.k8s.io/sig-storage/csi-resizer:v1.12.0 + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--leader-election=true" + - "--handle-volume-inuse-error=false" + - "--feature-gates=VolumeAttributesClass=true" + env: + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + volumes: + - name: socket-dir + emptyDir: + medium: "Memory" + # FIXED: Mount rook-ceph-csi-config instead of rook-ceph-nvmeof-*-config + # This ConfigMap contains the cluster configuration with monitor endpoints + # that Ceph-CSI needs to connect to the Ceph cluster + - name: ceph-csi-config + configMap: + name: rook-ceph-csi-config + items: + - key: csi-cluster-config-json + path: config.json + - name: keys-tmp-dir + emptyDir: + medium: "Memory" + # Only if nvmeof support encryption + - name: oidc-token + projected: + sources: + - serviceAccountToken: + path: oidc-token + expirationSeconds: 3600 + audience: ceph-csi-kms diff --git a/deploy/examples/csi/nvmeof/pvc.yaml b/deploy/examples/csi/nvmeof/pvc.yaml new file mode 100644 index 000000000..7578de8f7 --- /dev/null +++ b/deploy/examples/csi/nvmeof/pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nvmeof-external-volume + namespace: default +spec: + storageClassName: ceph-nvmeof + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 128Mi diff --git a/deploy/examples/csi/nvmeof/storageclass.yaml b/deploy/examples/csi/nvmeof/storageclass.yaml new file mode 100644 index 000000000..a686500c0 --- /dev/null +++ b/deploy/examples/csi/nvmeof/storageclass.yaml @@ -0,0 +1,31 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ceph-nvmeof +parameters: + clusterID: rook-ceph + pool: nvmeof + subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph + # Management API - talks to gateway to create subsystems/namespaces + nvmeofGatewayAddress: "172.30.47.120" + nvmeofGatewayPort: "5500" + # Data Plane - worker nodes connect here for actual I/O + # List ALL gateway pods for HA and multipath + listeners: | + [ + { + "address": "10.129.2.36", + "port": 4420, + "hostname": "rook-ceph-nvmeof-nvmeof-a" + } + ] + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + imageFormat: "2" + imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff +provisioner: nvmeof.csi.ceph.com +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: false diff --git a/deploy/examples/nvmeof-test.yaml b/deploy/examples/nvmeof-test.yaml new file mode 100644 index 000000000..58a4e1c95 --- /dev/null +++ b/deploy/examples/nvmeof-test.yaml @@ -0,0 +1,13 @@ +# This example is for Ceph v20 and above only + +apiVersion: ceph.rook.io/v1 +kind: CephNVMeOFGateway +metadata: + name: nvmeof + namespace: rook-ceph # namespace:cluster +spec: + image: quay.io/ceph/nvmeof:1.5 + pool: nvmeof + group: group-a + instances: 1 + hostNetwork: false diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 373e076d1..df4be3118 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -18,7 +18,7 @@ allowHostNetwork: false # set to true if running rook with the provider as host allowHostPorts: false priority: -allowedCapabilities: ["MKNOD"] +allowedCapabilities: ["MKNOD", "SYS_ADMIN", "NET_RAW"] allowHostIPC: true readOnlyRootFilesystem: false # drop all default privileges @@ -49,6 +49,7 @@ users: - system:serviceaccount:rook-ceph:rook-ceph-mgr # serviceaccount:namespace:cluster - system:serviceaccount:rook-ceph:rook-ceph-osd # serviceaccount:namespace:cluster - system:serviceaccount:rook-ceph:rook-ceph-rgw # serviceaccount:namespace:cluster + - system:serviceaccount:rook-ceph:rook-ceph-nvmeof # serviceaccount:namespace:cluster --- # scc for the CSI driver kind: SecurityContextConstraints @@ -97,6 +98,7 @@ users: - system:serviceaccount:rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:rook-csi-nfs-plugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:rook-csi-nfs-provisioner-sa # serviceaccount:namespace:operator + - system:serviceaccount:rook-ceph:nvmeof-csi-provisioner # serviceaccount:namespace:operator # Service accounts for the csi operator - system:serviceaccount:rook-ceph:ceph-csi-cephfs-ctrlplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-cephfs-nodeplugin-sa # serviceaccount:namespace:operator diff --git a/pkg/apis/ceph.rook.io/v1/nvmeof.go b/pkg/apis/ceph.rook.io/v1/nvmeof.go new file mode 100644 index 000000000..1d2b28118 --- /dev/null +++ b/pkg/apis/ceph.rook.io/v1/nvmeof.go @@ -0,0 +1,26 @@ +/* +Copyright 2024 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// IsHostNetwork returns true if host networking is enabled for this gateway. +// If not explicitly set on the gateway spec, it inherits from the cluster spec. +func (n *CephNVMeOFGateway) IsHostNetwork(c *ClusterSpec) bool { + if n.Spec.HostNetwork != nil { + return *n.Spec.HostNetwork + } + return c.Network.IsHost() +} diff --git a/pkg/apis/ceph.rook.io/v1/register.go b/pkg/apis/ceph.rook.io/v1/register.go index 1eebf7949..488e85ffd 100644 --- a/pkg/apis/ceph.rook.io/v1/register.go +++ b/pkg/apis/ceph.rook.io/v1/register.go @@ -65,6 +65,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &CephFilesystemList{}, &CephNFS{}, &CephNFSList{}, + &CephNVMeOFGateway{}, + &CephNVMeOFGatewayList{}, &CephObjectStore{}, &CephObjectStoreList{}, &CephObjectStoreUser{}, diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 3732c2fd0..aa5b267af 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -3929,3 +3929,125 @@ const ( // Always means the Ceph COSI driver will be deployed even if the object store is not present COSIDeploymentStrategyAlways COSIDeploymentStrategy = "Always" ) + +// +genclient +// +genclient:noStatus +// +kubebuilder:resource:shortName=nvmeof,path=cephnvmeofgateways +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +// +kubebuilder:subresource:status +// +// CephNVMeOFGateway represents a Ceph NVMe-oF Gateway +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type CephNVMeOFGateway struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + Spec NVMeOFGatewaySpec `json:"spec"` + // +optional + Status *NVMeOFGatewayStatus `json:"status,omitempty"` +} + +// NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway +type NVMeOFGatewayStatus struct { + Status `json:",inline"` + Cephx LocalCephxStatus `json:"cephx,omitempty"` +} + +// CephNVMeOFGatewayList represents a list of Ceph NVMe-oF Gateways +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type CephNVMeOFGatewayList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []CephNVMeOFGateway `json:"items"` +} + +// NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway +type NVMeOFGatewaySpec struct { + // Image is the container image to use for the NVMe-oF gateway daemon. + // For example, quay.io/ceph/nvmeof:1.5 + // +kubebuilder:validation:MinLength=1 + Image string `json:"image"` + + // The number of active gateway instances + // +kubebuilder:validation:Minimum=1 + Instances int `json:"instances"` + + // Pool is the RADOS pool where NVMe-oF configuration is stored + // +kubebuilder:validation:MinLength=1 + Pool string `json:"pool"` + + // Group is the gateway group name for high availability (ANA group) + // +kubebuilder:validation:MinLength=1 + Group string `json:"group"` + + // ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration + // If not specified, a default configuration will be generated + // +optional + // +kubebuilder:validation:MinLength=1 + ConfigMapRef string `json:"configMapRef,omitempty"` + + // NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration + // This allows users to override or add configuration options without needing to manage a ConfigMap + // +optional + NVMeOFConfig map[string]map[string]string `json:"nvmeofConfig,omitempty"` + + // The affinity to place the gateway pods + // +optional + Placement Placement `json:"placement,omitempty"` + + // The annotations-related configuration to add/set on each Pod related object. + // +optional + Annotations Annotations `json:"annotations,omitempty"` + + // The labels-related configuration to add/set on each Pod related object. + // +optional + Labels Labels `json:"labels,omitempty"` + + // Resources set resource requests and limits + // +optional + Resources v1.ResourceRequirements `json:"resources,omitempty"` + + // PriorityClassName sets the priority class on the pods + // +optional + PriorityClassName string `json:"priorityClassName,omitempty"` + + // Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied. + // +optional + HostNetwork *bool `json:"hostNetwork,omitempty"` + + // Ports configuration for the NVMe-oF gateway + // +optional + Ports *NVMeOFGatewayPorts `json:"ports,omitempty"` + + // A liveness-probe to verify that gateway has valid run-time state. + // If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + // +optional + LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"` +} + +// NVMeOFGatewayPorts represents the port configuration for NVMe-oF gateway +type NVMeOFGatewayPorts struct { + // IOPort is the port for NVMe-oF IO traffic (default: 4420) + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +optional + IOPort int32 `json:"ioPort,omitempty"` + + // GatewayPort is the port for the gateway service (default: 5500) + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +optional + GatewayPort int32 `json:"gatewayPort,omitempty"` + + // MonitorPort is the port for the monitor service (default: 5499) + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +optional + MonitorPort int32 `json:"monitorPort,omitempty"` + + // DiscoveryPort is the port for discovery service (default: 8009) + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +optional + DiscoveryPort int32 `json:"discoveryPort,omitempty"` +} diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index dc4fc5f2b..c480a40ab 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -1459,6 +1459,71 @@ func (in *CephNFSList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CephNVMeOFGateway) DeepCopyInto(out *CephNVMeOFGateway) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(NVMeOFGatewayStatus) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephNVMeOFGateway. +func (in *CephNVMeOFGateway) DeepCopy() *CephNVMeOFGateway { + if in == nil { + return nil + } + out := new(CephNVMeOFGateway) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CephNVMeOFGateway) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CephNVMeOFGatewayList) DeepCopyInto(out *CephNVMeOFGatewayList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CephNVMeOFGateway, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephNVMeOFGatewayList. +func (in *CephNVMeOFGatewayList) DeepCopy() *CephNVMeOFGatewayList { + if in == nil { + return nil + } + out := new(CephNVMeOFGatewayList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CephNVMeOFGatewayList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CephObjectRealm) DeepCopyInto(out *CephObjectRealm) { *out = *in @@ -3763,6 +3828,104 @@ func (in *NFSStatus) DeepCopy() *NFSStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NVMeOFGatewayPorts) DeepCopyInto(out *NVMeOFGatewayPorts) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NVMeOFGatewayPorts. +func (in *NVMeOFGatewayPorts) DeepCopy() *NVMeOFGatewayPorts { + if in == nil { + return nil + } + out := new(NVMeOFGatewayPorts) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NVMeOFGatewaySpec) DeepCopyInto(out *NVMeOFGatewaySpec) { + *out = *in + if in.NVMeOFConfig != nil { + in, out := &in.NVMeOFConfig, &out.NVMeOFConfig + *out = make(map[string]map[string]string, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } + in.Placement.DeepCopyInto(&out.Placement) + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(Annotations, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(Labels, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + in.Resources.DeepCopyInto(&out.Resources) + if in.HostNetwork != nil { + in, out := &in.HostNetwork, &out.HostNetwork + *out = new(bool) + **out = **in + } + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = new(NVMeOFGatewayPorts) + **out = **in + } + if in.LivenessProbe != nil { + in, out := &in.LivenessProbe, &out.LivenessProbe + *out = new(ProbeSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NVMeOFGatewaySpec. +func (in *NVMeOFGatewaySpec) DeepCopy() *NVMeOFGatewaySpec { + if in == nil { + return nil + } + out := new(NVMeOFGatewaySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NVMeOFGatewayStatus) DeepCopyInto(out *NVMeOFGatewayStatus) { + *out = *in + in.Status.DeepCopyInto(&out.Status) + out.Cephx = in.Cephx + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NVMeOFGatewayStatus. +func (in *NVMeOFGatewayStatus) DeepCopy() *NVMeOFGatewayStatus { + if in == nil { + return nil + } + out := new(NVMeOFGatewayStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NamedBlockPoolSpec) DeepCopyInto(out *NamedBlockPoolSpec) { *out = *in diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go index d500b7bc7..dc7c2faa4 100644 --- a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go +++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go @@ -39,6 +39,7 @@ type CephV1Interface interface { CephFilesystemMirrorsGetter CephFilesystemSubVolumeGroupsGetter CephNFSesGetter + CephNVMeOFGatewaysGetter CephObjectRealmsGetter CephObjectStoresGetter CephObjectStoreUsersGetter @@ -96,6 +97,10 @@ func (c *CephV1Client) CephNFSes(namespace string) CephNFSInterface { return newCephNFSes(c, namespace) } +func (c *CephV1Client) CephNVMeOFGateways(namespace string) CephNVMeOFGatewayInterface { + return newCephNVMeOFGateways(c, namespace) +} + func (c *CephV1Client) CephObjectRealms(namespace string) CephObjectRealmInterface { return newCephObjectRealms(c, namespace) } diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephnvmeofgateway.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephnvmeofgateway.go new file mode 100644 index 000000000..9eab2e2c4 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephnvmeofgateway.go @@ -0,0 +1,68 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +    http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + + cephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + scheme "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// CephNVMeOFGatewaysGetter has a method to return a CephNVMeOFGatewayInterface. +// A group's client should implement this interface. +type CephNVMeOFGatewaysGetter interface { + CephNVMeOFGateways(namespace string) CephNVMeOFGatewayInterface +} + +// CephNVMeOFGatewayInterface has methods to work with CephNVMeOFGateway resources. +type CephNVMeOFGatewayInterface interface { + Create(ctx context.Context, cephNVMeOFGateway *cephrookiov1.CephNVMeOFGateway, opts metav1.CreateOptions) (*cephrookiov1.CephNVMeOFGateway, error) + Update(ctx context.Context, cephNVMeOFGateway *cephrookiov1.CephNVMeOFGateway, opts metav1.UpdateOptions) (*cephrookiov1.CephNVMeOFGateway, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*cephrookiov1.CephNVMeOFGateway, error) + List(ctx context.Context, opts metav1.ListOptions) (*cephrookiov1.CephNVMeOFGatewayList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *cephrookiov1.CephNVMeOFGateway, err error) + CephNVMeOFGatewayExpansion +} + +// cephNVMeOFGateways implements CephNVMeOFGatewayInterface +type cephNVMeOFGateways struct { + *gentype.ClientWithList[*cephrookiov1.CephNVMeOFGateway, *cephrookiov1.CephNVMeOFGatewayList] +} + +// newCephNVMeOFGateways returns a CephNVMeOFGateways +func newCephNVMeOFGateways(c *CephV1Client, namespace string) *cephNVMeOFGateways { + return &cephNVMeOFGateways{ + gentype.NewClientWithList[*cephrookiov1.CephNVMeOFGateway, *cephrookiov1.CephNVMeOFGatewayList]( + "cephnvmeofgateways", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *cephrookiov1.CephNVMeOFGateway { return &cephrookiov1.CephNVMeOFGateway{} }, + func() *cephrookiov1.CephNVMeOFGatewayList { return &cephrookiov1.CephNVMeOFGatewayList{} }, + ), + } +} diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go index 6c6985e2a..5986773f2 100644 --- a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go +++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go @@ -72,6 +72,10 @@ func (c *FakeCephV1) CephNFSes(namespace string) v1.CephNFSInterface { return newFakeCephNFSes(c, namespace) } +func (c *FakeCephV1) CephNVMeOFGateways(namespace string) v1.CephNVMeOFGatewayInterface { + return newFakeCephNVMeOFGateways(c, namespace) +} + func (c *FakeCephV1) CephObjectRealms(namespace string) v1.CephObjectRealmInterface { return newFakeCephObjectRealms(c, namespace) } diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephnvmeofgateway.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephnvmeofgateway.go new file mode 100644 index 000000000..e09ce287e --- /dev/null +++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephnvmeofgateway.go @@ -0,0 +1,52 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +    http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + cephrookiov1 "github.com/rook/rook/pkg/client/clientset/versioned/typed/ceph.rook.io/v1" + gentype "k8s.io/client-go/gentype" +) + +// fakeCephNVMeOFGateways implements CephNVMeOFGatewayInterface +type fakeCephNVMeOFGateways struct { + *gentype.FakeClientWithList[*v1.CephNVMeOFGateway, *v1.CephNVMeOFGatewayList] + Fake *FakeCephV1 +} + +func newFakeCephNVMeOFGateways(fake *FakeCephV1, namespace string) cephrookiov1.CephNVMeOFGatewayInterface { + return &fakeCephNVMeOFGateways{ + gentype.NewFakeClientWithList[*v1.CephNVMeOFGateway, *v1.CephNVMeOFGatewayList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("cephnvmeofgateways"), + v1.SchemeGroupVersion.WithKind("CephNVMeOFGateway"), + func() *v1.CephNVMeOFGateway { return &v1.CephNVMeOFGateway{} }, + func() *v1.CephNVMeOFGatewayList { return &v1.CephNVMeOFGatewayList{} }, + func(dst, src *v1.CephNVMeOFGatewayList) { dst.ListMeta = src.ListMeta }, + func(list *v1.CephNVMeOFGatewayList) []*v1.CephNVMeOFGateway { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1.CephNVMeOFGatewayList, items []*v1.CephNVMeOFGateway) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go index 82f716799..cbd8ec6c4 100644 --- a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go @@ -40,6 +40,8 @@ type CephFilesystemSubVolumeGroupExpansion interface{} type CephNFSExpansion interface{} +type CephNVMeOFGatewayExpansion interface{} + type CephObjectRealmExpansion interface{} type CephObjectStoreExpansion interface{} diff --git a/pkg/client/informers/externalversions/ceph.rook.io/v1/cephnvmeofgateway.go b/pkg/client/informers/externalversions/ceph.rook.io/v1/cephnvmeofgateway.go new file mode 100644 index 000000000..0c18e160b --- /dev/null +++ b/pkg/client/informers/externalversions/ceph.rook.io/v1/cephnvmeofgateway.go @@ -0,0 +1,102 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +    http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + apiscephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + versioned "github.com/rook/rook/pkg/client/clientset/versioned" + internalinterfaces "github.com/rook/rook/pkg/client/informers/externalversions/internalinterfaces" + cephrookiov1 "github.com/rook/rook/pkg/client/listers/ceph.rook.io/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// CephNVMeOFGatewayInformer provides access to a shared informer and lister for +// CephNVMeOFGateways. +type CephNVMeOFGatewayInformer interface { + Informer() cache.SharedIndexInformer + Lister() cephrookiov1.CephNVMeOFGatewayLister +} + +type cephNVMeOFGatewayInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewCephNVMeOFGatewayInformer constructs a new informer for CephNVMeOFGateway type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewCephNVMeOFGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredCephNVMeOFGatewayInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredCephNVMeOFGatewayInformer constructs a new informer for CephNVMeOFGateway type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredCephNVMeOFGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CephV1().CephNVMeOFGateways(namespace).List(context.Background(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CephV1().CephNVMeOFGateways(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CephV1().CephNVMeOFGateways(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CephV1().CephNVMeOFGateways(namespace).Watch(ctx, options) + }, + }, + &apiscephrookiov1.CephNVMeOFGateway{}, + resyncPeriod, + indexers, + ) +} + +func (f *cephNVMeOFGatewayInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredCephNVMeOFGatewayInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *cephNVMeOFGatewayInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiscephrookiov1.CephNVMeOFGateway{}, f.defaultInformer) +} + +func (f *cephNVMeOFGatewayInformer) Lister() cephrookiov1.CephNVMeOFGatewayLister { + return cephrookiov1.NewCephNVMeOFGatewayLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go b/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go index 41fd56d9c..34666976b 100644 --- a/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go +++ b/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go @@ -46,6 +46,8 @@ type Interface interface { CephFilesystemSubVolumeGroups() CephFilesystemSubVolumeGroupInformer // CephNFSes returns a CephNFSInformer. CephNFSes() CephNFSInformer + // CephNVMeOFGateways returns a CephNVMeOFGatewayInformer. + CephNVMeOFGateways() CephNVMeOFGatewayInformer // CephObjectRealms returns a CephObjectRealmInformer. CephObjectRealms() CephObjectRealmInformer // CephObjectStores returns a CephObjectStoreInformer. @@ -126,6 +128,11 @@ func (v *version) CephNFSes() CephNFSInformer { return &cephNFSInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// CephNVMeOFGateways returns a CephNVMeOFGatewayInformer. +func (v *version) CephNVMeOFGateways() CephNVMeOFGatewayInformer { + return &cephNVMeOFGatewayInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // CephObjectRealms returns a CephObjectRealmInformer. func (v *version) CephObjectRealms() CephObjectRealmInformer { return &cephObjectRealmInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index 107983af1..21ef1a32e 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -75,6 +75,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephFilesystemSubVolumeGroups().Informer()}, nil case v1.SchemeGroupVersion.WithResource("cephnfses"): return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephNFSes().Informer()}, nil + case v1.SchemeGroupVersion.WithResource("cephnvmeofgateways"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephNVMeOFGateways().Informer()}, nil case v1.SchemeGroupVersion.WithResource("cephobjectrealms"): return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephObjectRealms().Informer()}, nil case v1.SchemeGroupVersion.WithResource("cephobjectstores"): diff --git a/pkg/client/listers/ceph.rook.io/v1/cephnvmeofgateway.go b/pkg/client/listers/ceph.rook.io/v1/cephnvmeofgateway.go new file mode 100644 index 000000000..e99251b97 --- /dev/null +++ b/pkg/client/listers/ceph.rook.io/v1/cephnvmeofgateway.go @@ -0,0 +1,70 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +    http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + cephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// CephNVMeOFGatewayLister helps list CephNVMeOFGateways. +// All objects returned here must be treated as read-only. +type CephNVMeOFGatewayLister interface { + // List lists all CephNVMeOFGateways in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*cephrookiov1.CephNVMeOFGateway, err error) + // CephNVMeOFGateways returns an object that can list and get CephNVMeOFGateways. + CephNVMeOFGateways(namespace string) CephNVMeOFGatewayNamespaceLister + CephNVMeOFGatewayListerExpansion +} + +// cephNVMeOFGatewayLister implements the CephNVMeOFGatewayLister interface. +type cephNVMeOFGatewayLister struct { + listers.ResourceIndexer[*cephrookiov1.CephNVMeOFGateway] +} + +// NewCephNVMeOFGatewayLister returns a new CephNVMeOFGatewayLister. +func NewCephNVMeOFGatewayLister(indexer cache.Indexer) CephNVMeOFGatewayLister { + return &cephNVMeOFGatewayLister{listers.New[*cephrookiov1.CephNVMeOFGateway](indexer, cephrookiov1.Resource("cephnvmeofgateway"))} +} + +// CephNVMeOFGateways returns an object that can list and get CephNVMeOFGateways. +func (s *cephNVMeOFGatewayLister) CephNVMeOFGateways(namespace string) CephNVMeOFGatewayNamespaceLister { + return cephNVMeOFGatewayNamespaceLister{listers.NewNamespaced[*cephrookiov1.CephNVMeOFGateway](s.ResourceIndexer, namespace)} +} + +// CephNVMeOFGatewayNamespaceLister helps list and get CephNVMeOFGateways. +// All objects returned here must be treated as read-only. +type CephNVMeOFGatewayNamespaceLister interface { + // List lists all CephNVMeOFGateways in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*cephrookiov1.CephNVMeOFGateway, err error) + // Get retrieves the CephNVMeOFGateway from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*cephrookiov1.CephNVMeOFGateway, error) + CephNVMeOFGatewayNamespaceListerExpansion +} + +// cephNVMeOFGatewayNamespaceLister implements the CephNVMeOFGatewayNamespaceLister +// interface. +type cephNVMeOFGatewayNamespaceLister struct { + listers.ResourceIndexer[*cephrookiov1.CephNVMeOFGateway] +} diff --git a/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go b/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go index a4d3eabcb..47b6a3046 100644 --- a/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go +++ b/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go @@ -106,6 +106,14 @@ type CephNFSListerExpansion interface{} // CephNFSNamespaceLister. type CephNFSNamespaceListerExpansion interface{} +// CephNVMeOFGatewayListerExpansion allows custom methods to be added to +// CephNVMeOFGatewayLister. +type CephNVMeOFGatewayListerExpansion interface{} + +// CephNVMeOFGatewayNamespaceListerExpansion allows custom methods to be added to +// CephNVMeOFGatewayNamespaceLister. +type CephNVMeOFGatewayNamespaceListerExpansion interface{} + // CephObjectRealmListerExpansion allows custom methods to be added to // CephObjectRealmLister. type CephObjectRealmListerExpansion interface{} diff --git a/pkg/operator/ceph/cr_manager.go b/pkg/operator/ceph/cr_manager.go index 60ae24ab3..42c7d5853 100644 --- a/pkg/operator/ceph/cr_manager.go +++ b/pkg/operator/ceph/cr_manager.go @@ -33,6 +33,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/file/mirror" "github.com/rook/rook/pkg/operator/ceph/file/subvolumegroup" "github.com/rook/rook/pkg/operator/ceph/nfs" + "github.com/rook/rook/pkg/operator/ceph/nvmeof" "github.com/rook/rook/pkg/operator/ceph/object" "github.com/rook/rook/pkg/operator/ceph/object/bucket" "github.com/rook/rook/pkg/operator/ceph/object/cosi" @@ -82,6 +83,7 @@ var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Conte nfs.Add, rbd.Add, client.Add, + nvmeof.Add, mirror.Add, Add, csi.Add, diff --git a/pkg/operator/ceph/nvmeof/connectionconfig.sh b/pkg/operator/ceph/nvmeof/connectionconfig.sh new file mode 100644 index 000000000..aa52958d0 --- /dev/null +++ b/pkg/operator/ceph/nvmeof/connectionconfig.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -xEeuo pipefail + +cat << EOF > /etc/ceph/ceph.conf +[global] +mon_host = ${ROOK_CEPH_MON_HOST} +log_to_stderr = true +keyring = /etc/ceph/keyring +EOF + +cp /tmp/ceph/keyring /etc/ceph + +chmod 444 /etc/ceph/ceph.conf +chmod 440 /etc/ceph/keyring + +# ARGS: all args are assumed to be 'ceph' CLI args +# Use "$@" to pass through Ceph CLI arguments (e.g., --connect-timeout, --mon-host, --keyring, etc.) +# This allows easy passthru of arguments set by Rook's methods for Ceph CLI options + +sed -e "s/@@POD_NAME@@/${GATEWAY_NAME}/g" \ + -e "s/@@ANA_GROUP@@/${ANA_GROUP}/g" \ + -e "s/@@POD_IP@@/${POD_IP}/g" \ + < /config/nvmeof.conf > /etc/ceph/nvmeof.conf + +ceph "$@" nvme-gw create "${GATEWAY_NAME}" "${POOL_NAME}" "${ANA_GROUP}" || true +ceph "$@" nvme-gw show ${POOL_NAME} ${ANA_GROUP} || true diff --git a/pkg/operator/ceph/nvmeof/controller.go b/pkg/operator/ceph/nvmeof/controller.go new file mode 100644 index 000000000..caa6eae2a --- /dev/null +++ b/pkg/operator/ceph/nvmeof/controller.go @@ -0,0 +1,587 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvmeof + +import ( + "context" + "fmt" + "reflect" + "strings" + + csiopv1 "github.com/ceph/ceph-csi-operator/api/v1" + "github.com/coreos/pkg/capnslog" + addonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/csiaddons/v1alpha1" + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/reporting" + "github.com/rook/rook/pkg/operator/k8sutil" + "gopkg.in/ini.v1" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + packageName = "ceph-nvmeof-gateway" + controllerName = packageName + "-controller" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", packageName) + +var objectsToWatch = []client.Object{ + &v1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: v1.SchemeGroupVersion.String()}}, + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.String()}}, +} + +var controllerTypeMeta = metav1.TypeMeta{ + Kind: reflect.TypeFor[cephv1.CephNVMeOFGateway]().Name(), + APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version), +} + +var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion + +type ReconcileCephNVMeOFGateway struct { + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + cephClusterSpec *cephv1.ClusterSpec + clusterInfo *cephclient.ClusterInfo + opManagerContext context.Context + opConfig opcontroller.OperatorConfig + recorder record.EventRecorder + shouldRotateCephxKeys bool +} + +// Add creates a new CephNVMeOFGateway Controller and adds it to the Manager. +func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error { + return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig)) +} + +func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { + return &ReconcileCephNVMeOFGateway{ + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + opManagerContext: opManagerContext, + opConfig: opConfig, + recorder: mgr.GetEventRecorderFor("rook-" + controllerName), + } +} + +func watchOwnedCoreObject[T client.Object](c controller.Controller, mgr manager.Manager, obj T) error { + return c.Watch( + source.Kind( + mgr.GetCache(), + obj, + handler.TypedEnqueueRequestForOwner[T]( + mgr.GetScheme(), + mgr.GetRESTMapper(), + &cephv1.CephNVMeOFGateway{}, + ), + opcontroller.WatchPredicateForNonCRDObject[T](&cephv1.CephNVMeOFGateway{TypeMeta: controllerTypeMeta}, mgr.GetScheme()), + ), + ) +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r}) + if err != nil { + return errors.Wrap(err, "failed to create controller") + } + logger.Info("successfully started") + + err = addonsv1alpha1.AddToScheme(mgr.GetScheme()) + if err != nil { + return errors.Wrap(err, "failed to add addonsv1alpha1 to scheme") + } + + err = csiopv1.AddToScheme(mgr.GetScheme()) + if err != nil { + return errors.Wrap(err, "failed to add csiopv1 to scheme") + } + + err = c.Watch( + source.Kind( + mgr.GetCache(), + &cephv1.CephNVMeOFGateway{TypeMeta: controllerTypeMeta}, + &handler.TypedEnqueueRequestForObject[*cephv1.CephNVMeOFGateway]{}, + opcontroller.WatchControllerPredicate[*cephv1.CephNVMeOFGateway](mgr.GetScheme()), + ), + ) + if err != nil { + return errors.Wrap(err, "failed to watch CephNVMeOFGateway CRD") + } + + for _, t := range objectsToWatch { + err = watchOwnedCoreObject(c, mgr, t) + if err != nil { + return errors.Wrapf(err, "failed to watch resource type %s", t.GetObjectKind().GroupVersionKind().String()) + } + } + + return nil +} + +// Reconcile reads the state of the cluster for a CephNVMeOFGateway object and makes changes based on the state read. +func (r *ReconcileCephNVMeOFGateway) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) { + defer opcontroller.RecoverAndLogException() + reconcileResponse, cephNVMeOFGateway, err := r.reconcile(request) + result, reportErr := reporting.ReportReconcileResult(logger, r.recorder, request, &cephNVMeOFGateway, reconcileResponse, err) + return result, reportErr +} + +func (r *ReconcileCephNVMeOFGateway) reconcile(request reconcile.Request) (reconcile.Result, cephv1.CephNVMeOFGateway, error) { + cephNVMeOFGateway := &cephv1.CephNVMeOFGateway{} + err := r.client.Get(r.opManagerContext, request.NamespacedName, cephNVMeOFGateway) + if err != nil { + if kerrors.IsNotFound(err) { + return reconcile.Result{}, *cephNVMeOFGateway, nil + } + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to get CephNVMeOFGateway") + } + + observedGeneration := cephNVMeOFGateway.ObjectMeta.Generation + + generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephNVMeOFGateway) + if err != nil { + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to add finalizer") + } + if generationUpdated { + return reconcile.Result{}, *cephNVMeOFGateway, nil + } + + if cephNVMeOFGateway.Status == nil { + cephxUninitialized := keyring.UninitializedCephxStatus() + err := r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, &cephxUninitialized, k8sutil.EmptyStatus) + if err != nil { + return opcontroller.ImmediateRetryResult, *cephNVMeOFGateway, errors.Wrapf(err, "failed set empty status") + } + cephNVMeOFGateway.Status = &cephv1.NVMeOFGatewayStatus{ + Status: cephv1.Status{}, + Cephx: cephv1.LocalCephxStatus{Daemon: cephxUninitialized}, + } + } + + cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) + + if !isReadyToReconcile { + if !cephNVMeOFGateway.GetDeletionTimestamp().IsZero() && !cephClusterExists { + err := opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephNVMeOFGateway) + if err != nil { + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to remove finalizer") + } + r.recorder.Event(cephNVMeOFGateway, v1.EventTypeNormal, string(cephv1.ReconcileSucceeded), "successfully removed finalizer") + return reconcile.Result{}, *cephNVMeOFGateway, nil + } + return reconcileResponse, *cephNVMeOFGateway, nil + } + r.cephClusterSpec = &cephCluster.Spec + + r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.cephClusterSpec) + if err != nil { + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to populate cluster info") + } + + if !cephNVMeOFGateway.GetDeletionTimestamp().IsZero() { + logger.Infof("deleting ceph nvmeof gateway %q", cephNVMeOFGateway.Name) + r.recorder.Eventf(cephNVMeOFGateway, v1.EventTypeNormal, string(cephv1.ReconcileStarted), "deleting CephNVMeOFGateway %q", cephNVMeOFGateway.Name) + + runningCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, config.MonType) + if err != nil { + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrapf(err, "failed to retrieve current ceph version") + } + r.clusterInfo.CephVersion = runningCephVersion + + err = opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephNVMeOFGateway) + if err != nil { + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to remove finalizer") + } + r.recorder.Event(cephNVMeOFGateway, v1.EventTypeNormal, string(cephv1.ReconcileSucceeded), "successfully removed finalizer") + return reconcile.Result{}, *cephNVMeOFGateway, nil + } + + runningCephVersion, desiredCephVersion, err := currentAndDesiredCephVersion( + r.opManagerContext, r.opConfig.Image, cephNVMeOFGateway.Namespace, controllerName, + k8sutil.NewOwnerInfo(cephNVMeOFGateway, r.scheme), r.context, r.cephClusterSpec, r.clusterInfo, + ) + if err != nil { + if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) { + logger.Info(opcontroller.OperatorNotInitializedMessage) + return opcontroller.WaitForRequeueIfOperatorNotInitialized, *cephNVMeOFGateway, nil + } + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to detect ceph version") + } + + if !cephCluster.Spec.External.Enable && !reflect.DeepEqual(*runningCephVersion, *desiredCephVersion) { + return opcontroller.WaitForRequeueIfCephClusterIsUpgrading, *cephNVMeOFGateway, + opcontroller.ErrorCephUpgradingRequeue(desiredCephVersion, runningCephVersion) + } + r.clusterInfo.CephVersion = *runningCephVersion + + if err := validateGateway(cephNVMeOFGateway); err != nil { + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrapf(err, "invalid configuration") + } + + r.shouldRotateCephxKeys, err = keyring.ShouldRotateCephxKeys(cephCluster.Spec.Security.CephX.Daemon, *runningCephVersion, + *desiredCephVersion, cephNVMeOFGateway.Status.Cephx.Daemon) + if err != nil { + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to determine if cephx keys should be rotated") + } + if r.shouldRotateCephxKeys { + logger.Infof("cephx keys will be rotated for %q", request.NamespacedName) + } + + _, err = r.reconcileCreateCephNVMeOFGateway(cephNVMeOFGateway) + if err != nil { + return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to create deployments") + } + + cephxStatus := keyring.UpdatedCephxStatus(r.shouldRotateCephxKeys, cephCluster.Spec.Security.CephX.Daemon, r.clusterInfo.CephVersion, cephNVMeOFGateway.Status.Cephx.Daemon) + err = r.updateStatus(observedGeneration, request.NamespacedName, &cephxStatus, k8sutil.ReadyStatus) + if err != nil { + logger.Errorf("failed to update status: %v", err) + return opcontroller.ImmediateRetryResult, *cephNVMeOFGateway, errors.Wrapf(err, "failed to update status") + } + + return reconcile.Result{}, *cephNVMeOFGateway, nil +} + +func (r *ReconcileCephNVMeOFGateway) reconcileCreateCephNVMeOFGateway(cephNVMeOFGateway *cephv1.CephNVMeOFGateway) (reconcile.Result, error) { + if r.cephClusterSpec.External.Enable { + _, err := opcontroller.ValidateCephVersionsBetweenLocalAndExternalClusters(r.context, r.clusterInfo) + if err != nil { + return reconcile.Result{}, errors.Wrap(err, "refusing to run new crd") + } + } + + listOps := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,app.kubernetes.io/part-of=%s", k8sutil.AppAttr, AppName, cephNVMeOFGateway.Name), + } + deployments, err := r.context.Clientset.AppsV1().Deployments(cephNVMeOFGateway.Namespace).List(r.opManagerContext, listOps) + if err != nil && !kerrors.IsNotFound(err) { + return reconcile.Result{}, errors.Wrapf(err, "failed to list deployments") + } + + currentGatewayCount := 0 + if deployments != nil { + currentGatewayCount = len(deployments.Items) + } + + if currentGatewayCount > cephNVMeOFGateway.Spec.Instances { + logger.Infof("scaling down from %d to %d", currentGatewayCount, cephNVMeOFGateway.Spec.Instances) + err := r.downCephNVMeOFGateway(cephNVMeOFGateway, currentGatewayCount) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to scale down") + } + } + + err = r.upCephNVMeOFGateway(cephNVMeOFGateway) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to update gateway") + } + + return reconcile.Result{}, nil +} + +func (r *ReconcileCephNVMeOFGateway) upCephNVMeOFGateway(cephNVMeOFGateway *cephv1.CephNVMeOFGateway) error { + for i := 0; i < cephNVMeOFGateway.Spec.Instances; i++ { + daemonID := k8sutil.IndexToName(i) + + var configMapName, configHash string + var err error + + if cephNVMeOFGateway.Spec.ConfigMapRef == "" { + configMapName, configHash, err = r.createConfigMap(cephNVMeOFGateway, daemonID) + if err != nil { + return errors.Wrapf(err, "failed to create configmap for %q", daemonID) + } + logger.Infof("configmap %q created/updated for nvmeof gateway %q instance %q with hash %q", configMapName, cephNVMeOFGateway.Name, daemonID, configHash) + } else { + configMapName = cephNVMeOFGateway.Spec.ConfigMapRef + configHash = "" // Will be empty for custom configmap + } + + deployment, err := r.makeDeployment(cephNVMeOFGateway, daemonID, configMapName, configHash) + if err != nil { + return errors.Wrapf(err, "failed to make deployment for %q", daemonID) + } + + err = controllerutil.SetControllerReference(cephNVMeOFGateway, deployment, r.scheme) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference for deployment %q", deployment.Name) + } + + _, err = k8sutil.CreateOrUpdateDeployment(r.opManagerContext, r.context.Clientset, deployment) + if err != nil { + return errors.Wrapf(err, "failed to create/update deployment for %q", daemonID) + } + + err = r.createCephNVMeOFService(cephNVMeOFGateway, daemonID) + if err != nil { + return errors.Wrapf(err, "failed to create service for %q", daemonID) + } + } + + return nil +} + +func (r *ReconcileCephNVMeOFGateway) downCephNVMeOFGateway(cephNVMeOFGateway *cephv1.CephNVMeOFGateway, currentCount int) error { + for i := cephNVMeOFGateway.Spec.Instances; i < currentCount; i++ { + daemonID := k8sutil.IndexToName(i) + name := instanceName(cephNVMeOFGateway, daemonID) + + err := r.context.Clientset.AppsV1().Deployments(cephNVMeOFGateway.Namespace).Delete(r.opManagerContext, name, metav1.DeleteOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to delete deployment %q", name) + } + + err = r.context.Clientset.CoreV1().Services(cephNVMeOFGateway.Namespace).Delete(r.opManagerContext, name, metav1.DeleteOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to delete service %q", name) + } + } + + return nil +} + +// getNVMeOFGatewayConfig generates a complete nvmeof.conf configuration file +// with all values filled in (no placeholders). User overrides from nvmeofConfig +// are merged on top of the default configuration. +func getNVMeOFGatewayConfig(poolName, podName, podIP, anaGroup string, userConfig map[string]map[string]string) (string, error) { + cfg := ini.Empty() + // Set default [gateway] section + gatewaySection, err := cfg.NewSection("gateway") + if err != nil { + return "", errors.Wrap(err, "failed to create gateway section") + } + gatewaySection.Key("name").SetValue(podName) + gatewaySection.Key("group").SetValue(anaGroup) + gatewaySection.Key("addr").SetValue(podIP) + gatewaySection.Key("port").SetValue("5500") + gatewaySection.Key("enable_auth").SetValue("False") + gatewaySection.Key("state_update_notify").SetValue("True") + gatewaySection.Key("state_update_timeout_in_msec").SetValue("2000") + gatewaySection.Key("state_update_interval_sec").SetValue("5") + gatewaySection.Key("enable_spdk_discovery_controller").SetValue("False") + gatewaySection.Key("encryption_key").SetValue("/etc/ceph/encryption.key") + gatewaySection.Key("rebalance_period_sec").SetValue("7") + gatewaySection.Key("max_gws_in_grp").SetValue("16") + gatewaySection.Key("max_ns_to_change_lb_grp").SetValue("8") + gatewaySection.Key("verify_listener_ip").SetValue("False") + gatewaySection.Key("enable_monitor_client").SetValue("True") + + // Set default [discovery] section + discoverySection, err := cfg.NewSection("discovery") + if err != nil { + return "", errors.Wrap(err, "failed to create discovery section") + } + discoverySection.Key("addr").SetValue("0.0.0.0") + discoverySection.Key("port").SetValue("8009") + + // Set default [ceph] section + cephSection, err := cfg.NewSection("ceph") + if err != nil { + return "", errors.Wrap(err, "failed to create ceph section") + } + cephSection.Key("id").SetValue("admin") + cephSection.Key("pool").SetValue(poolName) + cephSection.Key("config_file").SetValue("/etc/ceph/ceph.conf") + + // Set default [mtls] section + mtlsSection, err := cfg.NewSection("mtls") + if err != nil { + return "", errors.Wrap(err, "failed to create mtls section") + } + mtlsSection.Key("server_key").SetValue("./server.key") + mtlsSection.Key("client_key").SetValue("./client.key") + mtlsSection.Key("server_cert").SetValue("./server.crt") + mtlsSection.Key("client_cert").SetValue("./client.crt") + + // Set default [spdk] section + spdkSection, err := cfg.NewSection("spdk") + if err != nil { + return "", errors.Wrap(err, "failed to create spdk section") + } + spdkSection.Key("bdevs_per_cluster").SetValue("32") + spdkSection.Key("mem_size").SetValue("4096") + spdkSection.Key("tgt_path").SetValue("/usr/local/bin/nvmf_tgt") + spdkSection.Key("timeout").SetValue("60.0") + spdkSection.Key("rpc_socket").SetValue("/var/tmp/spdk.sock") + + // Set default [monitor] section + monitorSection, err := cfg.NewSection("monitor") + if err != nil { + return "", errors.Wrap(err, "failed to create monitor section") + } + monitorSection.Key("port").SetValue("5499") + + // Apply user overrides + for sectionName, options := range userConfig { + section := cfg.Section(sectionName) + if section == nil { + // Section doesn't exist, create it + var createErr error + section, createErr = cfg.NewSection(sectionName) + if createErr != nil { + return "", errors.Wrapf(createErr, "failed to create section %q", sectionName) + } + } + for key, value := range options { + section.Key(key).SetValue(value) + } + } + + // Write to string with proper formatting + var buf strings.Builder + _, err = cfg.WriteTo(&buf) + if err != nil { + return "", errors.Wrap(err, "failed to write config to string") + } + + return buf.String(), nil +} + +func (r *ReconcileCephNVMeOFGateway) generateConfigMap(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) (*v1.ConfigMap, error) { + poolName := nvmeof.Spec.Pool + anaGroup := nvmeof.Spec.Group + podName := instanceName(nvmeof, daemonID) + // Use placeholder that will be replaced at runtime with actual pod IP + // The init container will replace @@POD_IP@@ with the actual pod IP + podIP := "@@POD_IP@@" + + configContent, err := getNVMeOFGatewayConfig(poolName, podName, podIP, anaGroup, nvmeof.Spec.NVMeOFConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to generate nvmeof config") + } + data := map[string]string{ + "config": configContent, + } + + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-%s-config", nvmeof.Name, daemonID) + + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: nvmeof.Namespace, + Labels: getLabels(nvmeof, daemonID), + }, + Data: data, + } + + return configMap, nil +} + +func (r *ReconcileCephNVMeOFGateway) createConfigMap(cephNVMeOFGateway *cephv1.CephNVMeOFGateway, daemonID string) (string, string, error) { + configMap, err := r.generateConfigMap(cephNVMeOFGateway, daemonID) + if err != nil { + logger.Errorf("failed to generate configmap: %v", err) + return "", "", err + } + + err = controllerutil.SetControllerReference(cephNVMeOFGateway, configMap, r.scheme) + if err != nil { + logger.Errorf("failed to set owner reference: %v", err) + return "", "", errors.Wrapf(err, "failed to set owner reference for nvmeof configmap %q", configMap.Name) + } + + if _, err := r.context.Clientset.CoreV1().ConfigMaps(cephNVMeOFGateway.Namespace).Create(r.opManagerContext, configMap, metav1.CreateOptions{}); err != nil { + if !kerrors.IsAlreadyExists(err) { + logger.Errorf("failed to create configmap: %v", err) + return "", "", errors.Wrap(err, "failed to create nvmeof config map") + } + + if _, err = r.context.Clientset.CoreV1().ConfigMaps(cephNVMeOFGateway.Namespace).Update(r.opManagerContext, configMap, metav1.UpdateOptions{}); err != nil { + logger.Errorf("failed to update configmap: %v", err) + return "", "", errors.Wrap(err, "failed to update nvmeof config map") + } + } + + configHash := k8sutil.Hash(fmt.Sprintf("%v", configMap.Data)) + return configMap.Name, configHash, nil +} + +func validateGateway(g *cephv1.CephNVMeOFGateway) error { + if g.Spec.Instances < 1 { + return errors.New("at least one gateway instance is required") + } + + if g.Spec.Group == "" { + return errors.New("gateway group name is required") + } + + if g.Spec.Pool == "" { + return errors.New("pool name is required") + } + + return nil +} + +func (r *ReconcileCephNVMeOFGateway) updateStatus(observedGeneration int64, namespacedName types.NamespacedName, cephxStatus *cephv1.CephxStatus, status string) error { + nvmeof := &cephv1.CephNVMeOFGateway{} + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := r.client.Get(r.opManagerContext, namespacedName, nvmeof) + if err != nil { + if kerrors.IsNotFound(err) { + return nil + } + logger.Errorf("failed to get gateway: %v", err) + return errors.Wrapf(err, "failed to get for status update") + } + + if nvmeof.Status == nil { + nvmeof.Status = &cephv1.NVMeOFGatewayStatus{} + } + + nvmeof.Status.Phase = status + + if observedGeneration != k8sutil.ObservedGenerationNotAvailable { + nvmeof.Status.ObservedGeneration = observedGeneration + } + + if cephxStatus != nil { + nvmeof.Status.Cephx.Daemon = *cephxStatus + } + + if err := reporting.UpdateStatus(r.client, nvmeof); err != nil { + return errors.Wrapf(err, "failed to set status") + } + return nil + }) + if err != nil { + logger.Errorf("retry loop failed: %v", err) + return err + } + return nil +} diff --git a/pkg/operator/ceph/nvmeof/controller_test.go b/pkg/operator/ceph/nvmeof/controller_test.go new file mode 100644 index 000000000..2ce57ddfa --- /dev/null +++ b/pkg/operator/ceph/nvmeof/controller_test.go @@ -0,0 +1,712 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvmeof + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/coreos/pkg/capnslog" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/test" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + ini "gopkg.in/ini.v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var ( + name = "my-nvmeof" + namespace = "rook-ceph" +) + +func TestCephNVMeOFGatewayController(t *testing.T) { + ctx := context.TODO() + // Set DEBUG logging + capnslog.SetGlobalLogLevel(capnslog.DEBUG) + os.Setenv("ROOK_LOG_LEVEL", "DEBUG") + + // Register operator types with the runtime scheme. + testScheme := scheme.Scheme + testScheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephNVMeOFGateway{}) + testScheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) + + baseExecutor := func() *exectest.MockExecutor { + return &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + logger.Infof("mock execute: %s %v", command, args) + if command == "ceph" && args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + panic(fmt.Sprintf("unhandled command %s %v", command, args)) + }, + } + } + + successExecutor := func(t *testing.T) *exectest.MockExecutor { + t.Helper() + + return &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + logger.Infof("mock execute: %s %v", command, args) + if command == "ceph" { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + if args[0] == "nvme-gw" && args[1] == "create" { + return "", nil + } + if args[0] == "nvme-gw" && args[1] == "show" { + return "", nil + } + } + panic(fmt.Sprintf("unhandled command %s %v", command, args)) + }, + } + } + + newContext := func(executor *exectest.MockExecutor) *clusterd.Context { + clientset := test.New(t, 3) + return &clusterd.Context{ + Executor: executor, + RookClientset: rookclient.NewSimpleClientset(), + Clientset: clientset, + } + } + + baseCephNVMeOFGateway := func() *cephv1.CephNVMeOFGateway { + return &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Finalizers: []string{"cephnvmeofgateway.ceph.rook.io"}, + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + TypeMeta: controllerTypeMeta, + } + } + + cephClusterNotReady := func() *cephv1.CephCluster { + return &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, + }, + Status: cephv1.ClusterStatus{ + Phase: "", + CephStatus: &cephv1.CephStatus{ + Health: "", + }, + }, + } + } + + cephClusterReady := func(clusterCtx *clusterd.Context) *cephv1.CephCluster { + cephCluster := cephClusterNotReady() + cephCluster.Status.Phase = k8sutil.ReadyStatus + cephCluster.Status.CephStatus.Health = "HEALTH_OK" + + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := clusterCtx.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + if err != nil { + panic(fmt.Sprintf("should be no error: %v", err)) + } + + return cephCluster + } + + newControllerClient := func(objects ...runtime.Object) client.WithWatch { + return fake.NewClientBuilder().WithScheme(testScheme).WithRuntimeObjects(objects...).Build() + } + + newReconcile := func(clusterCtx *clusterd.Context, cl client.WithWatch) *ReconcileCephNVMeOFGateway { + return &ReconcileCephNVMeOFGateway{client: cl, scheme: testScheme, context: clusterCtx, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)} + } + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + + currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + return &version.Squid, &version.Squid, nil + } + + t.Run("error - no ceph cluster", func(t *testing.T) { + cCtx := newContext(baseExecutor()) + cl := newControllerClient(baseCephNVMeOFGateway()) + r := newReconcile(cCtx, cl) + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Greater(t, res.RequeueAfter, time.Duration(0)) + }) + + t.Run("error - ceph cluster not ready", func(t *testing.T) { + cCtx := newContext(baseExecutor()) + cl := newControllerClient(baseCephNVMeOFGateway(), cephClusterNotReady()) + r := newReconcile(cCtx, cl) + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Greater(t, res.RequeueAfter, time.Duration(0)) + }) + + t.Run("error - invalid gateway spec", func(t *testing.T) { + t.Run("instances less than 1 should error", func(t *testing.T) { + cCtx := newContext(successExecutor(t)) + nvmeof := baseCephNVMeOFGateway() + nvmeof.Spec.Instances = 0 + cl := newControllerClient(nvmeof, cephClusterReady(cCtx)) + r := newReconcile(cCtx, cl) + fakeRecorder := record.NewFakeRecorder(5) + r.recorder = fakeRecorder + + res, err := r.Reconcile(ctx, req) + assert.Error(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assert.Contains(t, err.Error(), "invalid configuration") + + assert.Len(t, fakeRecorder.Events, 1) + event := <-fakeRecorder.Events + assert.Contains(t, event, "ReconcileFailed") + assert.Contains(t, event, "invalid configuration") + }) + + t.Run("group empty should error", func(t *testing.T) { + cCtx := newContext(successExecutor(t)) + nvmeof := baseCephNVMeOFGateway() + nvmeof.Spec.Group = "" + cl := newControllerClient(nvmeof, cephClusterReady(cCtx)) + r := newReconcile(cCtx, cl) + fakeRecorder := record.NewFakeRecorder(5) + r.recorder = fakeRecorder + + res, err := r.Reconcile(ctx, req) + assert.Error(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assert.Contains(t, err.Error(), "invalid configuration") + + assert.Len(t, fakeRecorder.Events, 1) + event := <-fakeRecorder.Events + assert.Contains(t, event, "ReconcileFailed") + assert.Contains(t, event, "invalid configuration") + }) + }) + + assertCephNVMeOFGatewayReady := func(t *testing.T, r *ReconcileCephNVMeOFGateway, names ...string) { + t.Helper() + + if len(names) == 0 { + names = []string{name} + } + + for _, n := range names { + cephNVMeOFGateway := &cephv1.CephNVMeOFGateway{} + err := r.client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: n}, cephNVMeOFGateway) + assert.NoError(t, err) + assert.Equal(t, "Ready", cephNVMeOFGateway.Status.Phase, cephNVMeOFGateway) + } + } + + assertResourcesExist := func(t *testing.T, cCtx *clusterd.Context, names ...string) { + t.Helper() + + depNames := []string{} + deps, err := cCtx.Clientset.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + for _, dep := range deps.Items { + depNames = append(depNames, dep.Name) + } + assert.ElementsMatch(t, names, depNames) + + svcNames := []string{} + svcs, err := cCtx.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + for _, dep := range svcs.Items { + svcNames = append(svcNames, dep.Name) + } + assert.ElementsMatch(t, names, svcNames) + } + + t.Run("run one nvmeof gateway", func(t *testing.T) { + cCtx := newContext(successExecutor(t)) + cl := newControllerClient(baseCephNVMeOFGateway(), cephClusterReady(cCtx)) + r := newReconcile(cCtx, cl) + + t.Run("initial reconcile", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r) + assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a") + }) + + t.Run("double reconcile", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r) + assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a") + }) + }) + + t.Run("run multiple nvmeof gateways", func(t *testing.T) { + cCtx := newContext(successExecutor(t)) + nvmeof := baseCephNVMeOFGateway() + nvmeof.Spec.Instances = 3 + cl := newControllerClient(nvmeof, cephClusterReady(cCtx)) + r := newReconcile(cCtx, cl) + + t.Run("initial reconcile", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r) + assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c") + }) + + t.Run("double reconcile", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r) + assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c") + }) + }) + + t.Run("scale down nvmeof gateways", func(t *testing.T) { + t.Run("scale from 3 to 2 gateways", func(t *testing.T) { + cCtx := newContext(successExecutor(t)) + nvmeof := baseCephNVMeOFGateway() + nvmeof.Spec.Instances = 3 + cl := newControllerClient(nvmeof, cephClusterReady(cCtx)) + r := newReconcile(cCtx, cl) + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r) + assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c") + + err = cl.Get(ctx, client.ObjectKeyFromObject(nvmeof), nvmeof) + assert.NoError(t, err) + nvmeof.Spec.Instances = 2 + err = cl.Update(ctx, nvmeof, &client.UpdateOptions{}) + assert.NoError(t, err) + + res, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r) + assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b") + }) + + t.Run("scale from 3 to 1 gateway", func(t *testing.T) { + cCtx := newContext(successExecutor(t)) + nvmeof := baseCephNVMeOFGateway() + nvmeof.Spec.Instances = 3 + cl := newControllerClient(nvmeof, cephClusterReady(cCtx)) + r := newReconcile(cCtx, cl) + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r) + assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c") + + err = cl.Get(ctx, client.ObjectKeyFromObject(nvmeof), nvmeof) + assert.NoError(t, err) + nvmeof.Spec.Instances = 1 + err = cl.Update(ctx, nvmeof, &client.UpdateOptions{}) + assert.NoError(t, err) + + res, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r) + assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a") + }) + }) + + t.Run("multiple CephNVMeOFGateway clusters", func(t *testing.T) { + nvmeof1 := baseCephNVMeOFGateway() + nvmeof1.Spec.Instances = 3 + nvmeof2 := baseCephNVMeOFGateway() + nvmeof2.Spec.Instances = 2 + nvmeof2.Name = "nvmeof2" + + cCtx := newContext(successExecutor(t)) + cl := newControllerClient(nvmeof1, nvmeof2, cephClusterReady(cCtx)) + r := newReconcile(cCtx, cl) + + req2 := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "nvmeof2", + Namespace: namespace, + }, + } + + t.Run("reconcile first CephNVMeOFGateway cluster", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r, "my-nvmeof") // first cluster should be ready + assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c") + }) + + t.Run("reconcile second CephNVMeOFGateway cluster", func(t *testing.T) { + res, err := r.Reconcile(ctx, req2) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + assertCephNVMeOFGatewayReady(t, r, "my-nvmeof", "nvmeof2") + assertResourcesExist(t, cCtx, + "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c", + "rook-ceph-nvmeof-nvmeof2-a", "rook-ceph-nvmeof-nvmeof2-b", + ) + }) + + t.Run("scale down first CephNVMeOFGateway cluster (3 to 1)", func(t *testing.T) { + err := cl.Get(ctx, client.ObjectKeyFromObject(nvmeof1), nvmeof1) + assert.NoError(t, err) + nvmeof1.Spec.Instances = 1 + err = cl.Update(ctx, nvmeof1, &client.UpdateOptions{}) + assert.NoError(t, err) + + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + + assertCephNVMeOFGatewayReady(t, r, "my-nvmeof", "nvmeof2") + assertResourcesExist(t, cCtx, + "rook-ceph-nvmeof-my-nvmeof-a", + "rook-ceph-nvmeof-nvmeof2-a", "rook-ceph-nvmeof-nvmeof2-b", + ) + }) + }) +} + +func TestNVMeOFKeyRotation(t *testing.T) { + ctx := context.TODO() + var ( + name = "my-nvmeof" + namespace = "rook-ceph" + ) + // Set DEBUG logging + t.Setenv("ROOK_LOG_LEVEL", "DEBUG") + + // Mock version to avoid deployment updates (filesystem mirror pattern) + currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { + return &version.CephVersion{Major: 20, Minor: 2, Extra: 0}, &version.CephVersion{Major: 20, Minor: 2, Extra: 0}, nil + } + + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + TypeMeta: controllerTypeMeta, + } + + cephCluster := &cephv1.CephCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, + }, + Spec: cephv1.ClusterSpec{ + Security: cephv1.ClusterSecuritySpec{ + CephX: cephv1.ClusterCephxConfig{ + Daemon: cephv1.CephxConfig{}, + }, + }, + }, + Status: cephv1.ClusterStatus{ + Phase: k8sutil.ReadyStatus, + CephStatus: &cephv1.CephStatus{ + Health: "HEALTH_OK", + }, + }, + } + + object := []runtime.Object{ + nvmeof, + cephCluster, + } + + s := scheme.Scheme + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephNVMeOFGateway{}) + s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}) + + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { + if command == "ceph" { + if args[0] == "status" { + return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil + } + if args[0] == "nvme-gw" && args[1] == "create" { + return "", nil + } + if args[0] == "nvme-gw" && args[1] == "show" { + return "", nil + } + } + return "", nil + }, + } + clientset := test.New(t, 3) + c := &clusterd.Context{ + Executor: executor, + RookClientset: rookclient.NewSimpleClientset(), + Clientset: clientset, + } + + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + r := &ReconcileCephNVMeOFGateway{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)} + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + + t.Run("first reconcile", func(t *testing.T) { + secrets := map[string][]byte{ + "fsid": []byte(name), + "mon-secret": []byte("monsecret"), + "admin-secret": []byte("adminsecret"), + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-ceph-mon", + Namespace: namespace, + }, + Data: secrets, + Type: k8sutil.RookType, + } + _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + assert.NoError(t, err) + + _, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + + nvmeofResult := cephv1.CephNVMeOFGateway{} + err = cl.Get(ctx, req.NamespacedName, &nvmeofResult) + assert.NoError(t, err) + assert.Equal(t, uint32(1), nvmeofResult.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion) + }) + + t.Run("subsequent reconcile - retain cephx status", func(t *testing.T) { + r := &ReconcileCephNVMeOFGateway{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)} + _, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + nvmeofResult := cephv1.CephNVMeOFGateway{} + err = cl.Get(ctx, req.NamespacedName, &nvmeofResult) + assert.NoError(t, err) + assert.Equal(t, uint32(1), nvmeofResult.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion) + }) + + t.Run("brownfield reconcile - retain unknown cephx status", func(t *testing.T) { + nvmeofResult := cephv1.CephNVMeOFGateway{} + err := cl.Get(ctx, req.NamespacedName, &nvmeofResult) + assert.NoError(t, err) + nvmeofResult.Status.Cephx.Daemon = cephv1.CephxStatus{} + err = cl.Update(ctx, &nvmeofResult) + assert.NoError(t, err) + + _, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + + err = cl.Get(ctx, req.NamespacedName, &nvmeofResult) + assert.NoError(t, err) + assert.Equal(t, cephv1.CephxStatus{}, nvmeofResult.Status.Cephx.Daemon) + }) + + t.Run("rotate key - brownfield unknown status becomes known", func(t *testing.T) { + cluster := cephv1.CephCluster{} + err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster) + assert.NoError(t, err) + cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{ + KeyRotationPolicy: "KeyGeneration", + KeyGeneration: 2, + } + err = cl.Update(ctx, &cluster) + assert.NoError(t, err) + + _, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + + nvmeofResult := cephv1.CephNVMeOFGateway{} + err = cl.Get(ctx, req.NamespacedName, &nvmeofResult) + assert.NoError(t, err) + assert.Equal(t, uint32(2), nvmeofResult.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion) + }) + + t.Run("brownfield reconcile - no further rotation happens", func(t *testing.T) { + res, err := r.Reconcile(ctx, req) + assert.NoError(t, err) + assert.Equal(t, res.RequeueAfter, time.Duration(0)) + + nvmeofResult := cephv1.CephNVMeOFGateway{} + err = cl.Get(ctx, req.NamespacedName, &nvmeofResult) + assert.NoError(t, err) + assert.Equal(t, uint32(2), nvmeofResult.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion) + }) + + t.Run("rotate key - cephx status updated", func(t *testing.T) { + cluster := cephv1.CephCluster{} + err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster) + assert.NoError(t, err) + cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{ + KeyRotationPolicy: "KeyGeneration", + KeyGeneration: 3, + } + err = cl.Update(ctx, &cluster) + assert.NoError(t, err) + + _, err = r.Reconcile(ctx, req) + assert.NoError(t, err) + + nvmeofResult := cephv1.CephNVMeOFGateway{} + err = cl.Get(ctx, req.NamespacedName, &nvmeofResult) + assert.NoError(t, err) + assert.Equal(t, uint32(3), nvmeofResult.Status.Cephx.Daemon.KeyGeneration) + assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion) + }) +} + +func TestNVMeOFConfigGeneration(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + config, err := getNVMeOFGatewayConfig("pool-a", "pod-a", "10.0.0.1", "ana-a", nil) + assert.NoError(t, err) + + cfg, err := ini.Load([]byte(config)) + assert.NoError(t, err) + assert.Equal(t, "pod-a", cfg.Section("gateway").Key("name").String()) + assert.Equal(t, "ana-a", cfg.Section("gateway").Key("group").String()) + assert.Equal(t, "10.0.0.1", cfg.Section("gateway").Key("addr").String()) + assert.Equal(t, "5500", cfg.Section("gateway").Key("port").String()) + assert.Equal(t, "0.0.0.0", cfg.Section("discovery").Key("addr").String()) + assert.Equal(t, "8009", cfg.Section("discovery").Key("port").String()) + assert.Equal(t, "admin", cfg.Section("ceph").Key("id").String()) + assert.Equal(t, "pool-a", cfg.Section("ceph").Key("pool").String()) + assert.Equal(t, "4096", cfg.Section("spdk").Key("mem_size").String()) + assert.Equal(t, "5499", cfg.Section("monitor").Key("port").String()) + }) + + t.Run("overrides and new section", func(t *testing.T) { + userConfig := map[string]map[string]string{ + "gateway": { + "port": "6600", + "enable_auth": "True", + }, + "ceph": { + "pool": "pool-b", + }, + "custom": { + "foo": "bar", + }, + } + config, err := getNVMeOFGatewayConfig("pool-a", "pod-a", "10.0.0.1", "ana-a", userConfig) + assert.NoError(t, err) + + cfg, err := ini.Load([]byte(config)) + assert.NoError(t, err) + assert.Equal(t, "6600", cfg.Section("gateway").Key("port").String()) + assert.Equal(t, "True", cfg.Section("gateway").Key("enable_auth").String()) + assert.Equal(t, "pool-b", cfg.Section("ceph").Key("pool").String()) + assert.Equal(t, "bar", cfg.Section("custom").Key("foo").String()) + }) +} + +func TestNVMeOFConfigMapGeneration(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Pool: "pool-a", + Group: "group-a", + Instances: 1, + NVMeOFConfig: map[string]map[string]string{ + "gateway": { + "port": "5511", + }, + }, + }, + TypeMeta: controllerTypeMeta, + } + + r := &ReconcileCephNVMeOFGateway{} + configMap, err := r.generateConfigMap(nvmeof, "a") + assert.NoError(t, err) + assert.Equal(t, "rook-ceph-nvmeof-my-nvmeof-a-config", configMap.Name) + assert.Equal(t, "rook-ceph", configMap.Namespace) + assert.Contains(t, configMap.Data, "config") + + cfg, err := ini.Load([]byte(configMap.Data["config"])) + assert.NoError(t, err) + assert.Equal(t, instanceName(nvmeof, "a"), cfg.Section("gateway").Key("name").String()) + assert.Equal(t, "@@POD_IP@@", cfg.Section("gateway").Key("addr").String()) + assert.Equal(t, "group-a", cfg.Section("gateway").Key("group").String()) + assert.Equal(t, "5511", cfg.Section("gateway").Key("port").String()) + assert.Equal(t, "pool-a", cfg.Section("ceph").Key("pool").String()) +} diff --git a/pkg/operator/ceph/nvmeof/spec.go b/pkg/operator/ceph/nvmeof/spec.go new file mode 100644 index 000000000..06b415927 --- /dev/null +++ b/pkg/operator/ceph/nvmeof/spec.go @@ -0,0 +1,474 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvmeof + +import ( + _ "embed" + "fmt" + "strings" + + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/k8sutil" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + AppName = "rook-ceph-nvmeof" + nvmeofIOPort = 4420 + nvmeofGatewayPort = 5500 + nvmeofMonitorPort = 5499 + nvmeofDiscoveryPort = 8009 + defaultNVMeOFImage = "quay.io/ceph/nvmeof:1.5" + configKey = "config" + serviceAccountName = "rook-ceph-nvmeof" +) + +// getPorts returns the configured ports with defaults +func getPorts(nvmeof *cephv1.CephNVMeOFGateway) (ioPort, gatewayPort, monitorPort, discoveryPort int32) { + ioPort = nvmeofIOPort + gatewayPort = nvmeofGatewayPort + monitorPort = nvmeofMonitorPort + discoveryPort = nvmeofDiscoveryPort + + if nvmeof.Spec.Ports != nil { + if nvmeof.Spec.Ports.IOPort != 0 { + ioPort = nvmeof.Spec.Ports.IOPort + } + if nvmeof.Spec.Ports.GatewayPort != 0 { + gatewayPort = nvmeof.Spec.Ports.GatewayPort + } + if nvmeof.Spec.Ports.MonitorPort != 0 { + monitorPort = nvmeof.Spec.Ports.MonitorPort + } + if nvmeof.Spec.Ports.DiscoveryPort != 0 { + discoveryPort = nvmeof.Spec.Ports.DiscoveryPort + } + } + + return ioPort, gatewayPort, monitorPort, discoveryPort +} + +//go:embed connectionconfig.sh +var connectionConfigScript string + +func (r *ReconcileCephNVMeOFGateway) generateCephNVMeOFService(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) *v1.Service { + labels := getLabels(nvmeof, daemonID) + serviceName := instanceName(nvmeof, daemonID) + ioPort, gatewayPort, monitorPort, discoveryPort := getPorts(nvmeof) + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: nvmeof.Namespace, + Labels: labels, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Ports: []v1.ServicePort{ + { + Name: "io", + Port: ioPort, + TargetPort: intstr.FromInt(int(ioPort)), + Protocol: v1.ProtocolTCP, + }, + { + Name: "gateway", + Port: gatewayPort, + TargetPort: intstr.FromInt(int(gatewayPort)), + Protocol: v1.ProtocolTCP, + }, + { + Name: "monitor", + Port: monitorPort, + TargetPort: intstr.FromInt(int(monitorPort)), + Protocol: v1.ProtocolTCP, + }, + { + Name: "discovery", + Port: discoveryPort, + TargetPort: intstr.FromInt(int(discoveryPort)), + Protocol: v1.ProtocolTCP, + }, + }, + }, + } + + hostNetwork := nvmeof.IsHostNetwork(r.cephClusterSpec) + if hostNetwork { + svc.Spec.ClusterIP = v1.ClusterIPNone + } + + return svc +} + +func (r *ReconcileCephNVMeOFGateway) createCephNVMeOFService(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) error { + s := r.generateCephNVMeOFService(nvmeof, daemonID) + + err := controllerutil.SetControllerReference(nvmeof, s, r.scheme) + if err != nil { + logger.Errorf("failed to set owner reference: %v", err) + return errors.Wrapf(err, "failed to set owner reference to ceph nvmeof gateway %q", s) + } + + svc, err := r.context.Clientset.CoreV1().Services(nvmeof.Namespace).Create(r.opManagerContext, s, metav1.CreateOptions{}) + if err != nil { + if !kerrors.IsAlreadyExists(err) { + logger.Errorf("failed to create service: %v", err) + return errors.Wrap(err, "failed to create nvmeof gateway service") + } + logger.Infof("ceph nvmeof gateway service already created") + return nil + } + + _, _, _, discoveryPort := getPorts(nvmeof) + logger.Infof("ceph nvmeof gateway service running at %s:%d", svc.Spec.ClusterIP, discoveryPort) + return nil +} + +func (r *ReconcileCephNVMeOFGateway) makeDeployment(nvmeof *cephv1.CephNVMeOFGateway, daemonID, configMapName, configHash string) (*apps.Deployment, error) { + if r == nil { + logger.Errorf("receiver is nil") + return nil, errors.New("receiver is nil") + } + if r.clusterInfo == nil { + logger.Errorf("clusterInfo is nil") + return nil, errors.New("clusterInfo is nil") + } + if r.clusterInfo.CephVersion.String() == "" { + logger.Errorf("CephVersion is empty") + return nil, errors.New("CephVersion is empty") + } + + resourceName := instanceName(nvmeof, daemonID) + deployment := &apps.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: nvmeof.Namespace, + Labels: getLabels(nvmeof, daemonID), + }, + } + + hostNetwork := nvmeof.IsHostNetwork(r.cephClusterSpec) + + k8sutil.AddRookVersionLabelToDeployment(deployment) + controller.AddCephVersionLabelToDeployment(r.clusterInfo.CephVersion, deployment) + nvmeof.Spec.Annotations.ApplyToObjectMeta(&deployment.ObjectMeta) + nvmeof.Spec.Labels.ApplyToObjectMeta(&deployment.ObjectMeta) + + cephConfigVol, cephConfigMount := controller.ConfGeneratedInPodVolumeAndMount() + gatewayConfigVol, gatewayConfigMount := gatewayConfigVolumeAndMount(configMapName) + + initContainer := r.createCephConfigInitContainer(nvmeof, daemonID, gatewayConfigMount) + daemonContainer := r.daemonContainer(nvmeof, cephConfigMount) + + gatewayName := instanceName(nvmeof, daemonID) + podSpec := v1.PodSpec{ + InitContainers: []v1.Container{ + initContainer, + }, + Containers: []v1.Container{ + daemonContainer, + }, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: []v1.Volume{ + cephConfigVol, + adminKeyringVolume(), + gatewayConfigVol, + }, + HostNetwork: hostNetwork, + PriorityClassName: nvmeof.Spec.PriorityClassName, + SecurityContext: &v1.PodSecurityContext{}, + ServiceAccountName: serviceAccountName, + } + + // Ensure a stable hostname inside the pod (does NOT make the pod IP stable). + // This matches the gateway instance name used elsewhere by the operator. + if errs := validation.IsDNS1123Label(gatewayName); len(errs) == 0 { + podSpec.Hostname = gatewayName + } else { + logger.Warningf("not setting pod hostname %q (must be a DNS-1123 label): %v", gatewayName, errs) + } + + k8sutil.AddUnreachableNodeToleration(&podSpec) + + if hostNetwork { + podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } + nvmeof.Spec.Placement.ApplyToPodSpec(&podSpec) + + podTemplateSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Labels: getLabels(nvmeof, daemonID), + Annotations: map[string]string{ + "config-hash": configHash, + }, + }, + Spec: podSpec, + } + + if hostNetwork { + podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if r.cephClusterSpec.Network.IsMultus() { + if err := k8sutil.ApplyMultus(r.clusterInfo.Namespace, &r.cephClusterSpec.Network, &podTemplateSpec.ObjectMeta); err != nil { + logger.Errorf("failed to apply Multus configuration: %v", err) + return nil, err + } + } + + nvmeof.Spec.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta) + nvmeof.Spec.Labels.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta) + + replicas := int32(1) + revisionHistoryLimit := controller.RevisionHistoryLimit() + deployment.Spec = apps.DeploymentSpec{ + RevisionHistoryLimit: revisionHistoryLimit, + Selector: &metav1.LabelSelector{MatchLabels: getLabels(nvmeof, daemonID)}, + Template: podTemplateSpec, + Replicas: &replicas, + } + + return deployment, nil +} + +// createCephConfigInitContainer creates a minimal init container that generates +// /etc/ceph/ceph.conf, copies the admin keyring, and copies nvmeof.conf from the ConfigMap. +// This is needed for the gateway to connect to the Ceph cluster. +func (r *ReconcileCephNVMeOFGateway) createCephConfigInitContainer(nvmeof *cephv1.CephNVMeOFGateway, daemonID string, gatewayConfigMount v1.VolumeMount) v1.Container { + _, cephConfigMount := controller.ConfGeneratedInPodVolumeAndMount() + + cephImage := r.cephClusterSpec.CephVersion.Image + imagePullPolicy := controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy) + + // Build Ceph CLI arguments using Rook's helper functions + // Use AdminFlags since the init container uses admin keyring for Ceph commands + // Override the keyring path to use /etc/ceph/keyring (where the script copies it) + cephArgs := controller.AdminFlags(r.clusterInfo) + // Replace the keyring path with the one used by the init container script + for i, arg := range cephArgs { + if strings.HasPrefix(arg, "--keyring=") { + cephArgs[i] = "--keyring=/etc/ceph/keyring" + break + } + } + + // Use the embedded script and pass Ceph CLI args via "$@" + script := connectionConfigScript + + gatewayName := instanceName(nvmeof, daemonID) + poolName := nvmeof.Spec.Pool + anaGroup := nvmeof.Spec.Group + + // Build environment variables using Rook's helper functions + // DaemonEnvVars already includes StoredMonHostEnvVars() which provides ROOK_CEPH_MON_HOST + envVars := controller.DaemonEnvVars(r.cephClusterSpec) + envVars = append(envVars, + v1.EnvVar{ + Name: "GATEWAY_NAME", + Value: gatewayName, + }, + v1.EnvVar{ + Name: "POOL_NAME", + Value: poolName, + }, + v1.EnvVar{ + Name: "ANA_GROUP", + Value: anaGroup, + }, + k8sutil.PodIPEnvVar("POD_IP"), + ) + + privileged := true + container := v1.Container{ + Name: "generate-ceph-conf", + Image: cephImage, + ImagePullPolicy: imagePullPolicy, + Command: []string{ + "/bin/bash", + "-c", + script, + }, + Args: cephArgs, + Env: envVars, + VolumeMounts: []v1.VolumeMount{ + adminKeyringVolumeMount(), + cephConfigMount, + gatewayConfigMount, + }, + Resources: nvmeof.Spec.Resources, + SecurityContext: &v1.SecurityContext{ + Privileged: &privileged, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{ + "SYS_ADMIN", + }, + Drop: []v1.Capability{ + "NET_RAW", + }, + }, + }, + } + return container +} + +func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGateway, cephConfigMount v1.VolumeMount) v1.Container { + privileged := true + container := v1.Container{ + Name: "nvmeof-gateway", + Args: []string{ + "-c", + "/etc/ceph/nvmeof.conf", + }, + + Image: getNVMeOFImage(nvmeof), + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: []v1.VolumeMount{ + cephConfigMount, + }, + Env: func() []v1.EnvVar { + envVars := controller.DaemonEnvVars(r.cephClusterSpec) + // Build CEPH_ARGS using DefaultFlags which includes fsid, keyring, logging flags, and mon host flags + cephArgs := cephconfig.DefaultFlags(r.clusterInfo.FSID, "/etc/ceph/keyring") + cephArgsStr := strings.Join(cephArgs, " ") + logger.Infof("CEPH_ARGS for nvmeof gateway: %s", cephArgsStr) + envVars = append(envVars, v1.EnvVar{ + Name: "CEPH_ARGS", + Value: cephArgsStr, + }) + return envVars + }(), + + Ports: func() []v1.ContainerPort { + ioPort, gatewayPort, monitorPort, discoveryPort := getPorts(nvmeof) + return []v1.ContainerPort{ + { + Name: "io", + ContainerPort: ioPort, + Protocol: v1.ProtocolTCP, + }, + { + Name: "gateway", + ContainerPort: gatewayPort, + Protocol: v1.ProtocolTCP, + }, + { + Name: "monitor", + ContainerPort: monitorPort, + Protocol: v1.ProtocolTCP, + }, + { + Name: "discovery", + ContainerPort: discoveryPort, + Protocol: v1.ProtocolTCP, + }, + } + }(), + + Resources: nvmeof.Spec.Resources, + SecurityContext: &v1.SecurityContext{ + Privileged: &privileged, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{ + "SYS_ADMIN", + }, + Drop: []v1.Capability{ + "NET_RAW", + }, + }, + }, + } + + if nvmeof.Spec.LivenessProbe != nil && !nvmeof.Spec.LivenessProbe.Disabled && nvmeof.Spec.LivenessProbe.Probe == nil { + container.LivenessProbe = r.defaultLivenessProbe(nvmeof) + } + result := cephconfig.ConfigureLivenessProbe(container, nvmeof.Spec.LivenessProbe) + return result +} + +func (r *ReconcileCephNVMeOFGateway) defaultLivenessProbe(nvmeof *cephv1.CephNVMeOFGateway) *v1.Probe { + ioPort, _, _, _ := getPorts(nvmeof) + return controller.GenerateLivenessProbeTcpPort(ioPort, 10) +} + +func getLabels(n *cephv1.CephNVMeOFGateway, daemonID string) map[string]string { + return controller.CephDaemonAppLabels( + AppName, n.Namespace, "nvmeof", n.Name+"-"+daemonID, n.Name, + "cephnvmeofgateways.ceph.rook.io", true, + ) +} + +func instanceName(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) string { + return fmt.Sprintf("%s-%s-%s", AppName, nvmeof.Name, daemonID) +} + +// getNVMeOFImage returns the image to use for the NVMe-oF gateway. +// If the image is specified in the spec, it is used; otherwise, the default image is returned. +func getNVMeOFImage(nvmeof *cephv1.CephNVMeOFGateway) string { + if nvmeof.Spec.Image != "" { + return nvmeof.Spec.Image + } + return defaultNVMeOFImage +} + +func gatewayConfigVolumeAndMount(configConfigMap string) (v1.Volume, v1.VolumeMount) { + // Mount to /config temporarily - init container will copy to /etc/ceph/nvmeof.conf + cfgDir := "/config" + cfgVolName := "gateway-config" + configMapSource := &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{Name: configConfigMap}, + Items: []v1.KeyToPath{{Key: configKey, Path: "nvmeof.conf"}}, + } + v := v1.Volume{Name: cfgVolName, VolumeSource: v1.VolumeSource{ConfigMap: configMapSource}} + m := v1.VolumeMount{Name: cfgVolName, MountPath: cfgDir, ReadOnly: true} + return v, m +} + +func adminKeyringVolume() v1.Volume { + mode := int32(292) + return v1.Volume{ + Name: "ceph-admin-keyring", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "rook-ceph-admin-keyring", + Items: []v1.KeyToPath{ + { + Key: "keyring", + Path: "keyring", + Mode: &mode, + }, + }, + }, + }, + } +} + +func adminKeyringVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: "ceph-admin-keyring", + MountPath: "/tmp/ceph/", + ReadOnly: true, + } +} diff --git a/pkg/operator/ceph/nvmeof/spec_test.go b/pkg/operator/ceph/nvmeof/spec_test.go new file mode 100644 index 000000000..3cef9141e --- /dev/null +++ b/pkg/operator/ceph/nvmeof/spec_test.go @@ -0,0 +1,536 @@ +/* +Copyright 2025 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvmeof + +import ( + "fmt" + "strings" + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" + "github.com/rook/rook/pkg/client/clientset/versioned/scheme" + "github.com/rook/rook/pkg/clusterd" + cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" + cephTest "github.com/rook/rook/pkg/operator/ceph/test" + cephver "github.com/rook/rook/pkg/operator/ceph/version" + optest "github.com/rook/rook/pkg/operator/test" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func newDeploymentSpecTest(t *testing.T) (*ReconcileCephNVMeOFGateway, string) { + clientset := optest.New(t, 1) + c := &clusterd.Context{ + Executor: &exectest.MockExecutor{}, + RookClientset: rookclient.NewSimpleClientset(), + Clientset: clientset, + } + + s := scheme.Scheme + object := []runtime.Object{ + &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + TypeMeta: controllerTypeMeta, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + }, + } + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + + r := &ReconcileCephNVMeOFGateway{ + client: cl, + scheme: scheme.Scheme, + context: c, + clusterInfo: &cephclient.ClusterInfo{ + FSID: "myfsid", + CephVersion: cephver.Squid, + }, + cephClusterSpec: &cephv1.ClusterSpec{ + CephVersion: cephv1.CephVersionSpec{ + Image: "quay.io/ceph/ceph:v19", + }, + }, + } + + configHash := "dcb0d2f5f5e86ec4929d8243cd640b8154165f8ff9b89809964fc7993e9b0101" + + return r, configHash +} + +func TestDeploymentSpec(t *testing.T) { + t.Run("basic", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 3, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI), + }, + }, + PriorityClassName: "my-priority-class", + }, + } + + r, configHash := newDeploymentSpecTest(t) + + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + assert.NotEmpty(t, d.Spec.Template.Annotations) + assert.Equal(t, configHash, d.Spec.Template.Annotations["config-hash"]) + + // Verify deployment labels contain Ceph requirements + daemonID := nvmeof.Name + "-0" + cephTest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels, + "nvmeof", daemonID, AppName, nvmeof.Namespace, nvmeof.Name, + "cephnvmeofgateways.ceph.rook.io", "ceph-nvmeof") + + podTemplate := optest.NewPodTemplateSpecTester(t, &d.Spec.Template) + podTemplate.RunFullSuite( + AppName, + optest.ResourceLimitExpectations{ + CPUResourceLimit: "500", + MemoryResourceLimit: "1Ki", + CPUResourceRequest: "200", + MemoryResourceRequest: "512", + }, + ) + assert.Equal(t, "my-priority-class", d.Spec.Template.Spec.PriorityClassName) + assert.Equal(t, serviceAccountName, d.Spec.Template.Spec.ServiceAccountName) + }) + + t.Run("basic with default liveness-probe", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 3, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI), + }, + }, + PriorityClassName: "my-priority-class", + LivenessProbe: &cephv1.ProbeSpec{ + Disabled: false, + }, + }, + } + + r, configHash := newDeploymentSpecTest(t) + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + assert.NotEmpty(t, d.Spec.Template.Annotations) + + var nvmeofCont *v1.Container = nil + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == "nvmeof-gateway" { + nvmeofCont = &d.Spec.Template.Spec.Containers[i] + break + } + } + assert.NotNil(t, nvmeofCont) + assert.NotNil(t, nvmeofCont.LivenessProbe) + assert.Equal(t, nvmeofCont.LivenessProbe.InitialDelaySeconds, int32(10)) + assert.Equal(t, nvmeofCont.LivenessProbe.FailureThreshold, int32(10)) + assert.GreaterOrEqual(t, nvmeofCont.LivenessProbe.TimeoutSeconds, int32(5)) + }) + + t.Run("with host network", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + } + + r, configHash := newDeploymentSpecTest(t) + r.cephClusterSpec.Network = cephv1.NetworkSpec{ + HostNetwork: true, + } + + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + assert.True(t, d.Spec.Template.Spec.HostNetwork) + assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy) + + svc := r.generateCephNVMeOFService(nvmeof, "0") + assert.Equal(t, v1.ClusterIPNone, svc.Spec.ClusterIP) + }) + + t.Run("with custom configmap ref", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + ConfigMapRef: "custom-configmap", + }, + } + + r, configHash := newDeploymentSpecTest(t) + + configMapName := "custom-configmap" + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + assert.NotEmpty(t, d.Spec.Template.Annotations) + + var gatewayConfigVol *v1.Volume = nil + for i := range d.Spec.Template.Spec.Volumes { + if d.Spec.Template.Spec.Volumes[i].Name == "gateway-config" { + gatewayConfigVol = &d.Spec.Template.Spec.Volumes[i] + break + } + } + assert.NotNil(t, gatewayConfigVol) + assert.NotNil(t, gatewayConfigVol.ConfigMap) + assert.Equal(t, "custom-configmap", gatewayConfigVol.ConfigMap.Name) + }) + + t.Run("init container configuration", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + } + + r, configHash := newDeploymentSpecTest(t) + + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + + assert.Len(t, d.Spec.Template.Spec.InitContainers, 1) + initCont := d.Spec.Template.Spec.InitContainers[0] + assert.Equal(t, "generate-ceph-conf", initCont.Name) + assert.Equal(t, r.cephClusterSpec.CephVersion.Image, initCont.Image) + assert.Equal(t, []string{"/bin/bash", "-c", connectionConfigScript}, initCont.Command) + assert.Contains(t, initCont.Args, "--keyring=/etc/ceph/keyring") + + assertEnvVar(t, initCont.Env, "GATEWAY_NAME", instanceName(nvmeof, "0")) + assertEnvVar(t, initCont.Env, "POOL_NAME", nvmeof.Spec.Pool) + assertEnvVar(t, initCont.Env, "ANA_GROUP", nvmeof.Spec.Group) + assertEnvVarPresent(t, initCont.Env, "POD_IP") + + volumeMountNames := []string{} + for _, vm := range initCont.VolumeMounts { + volumeMountNames = append(volumeMountNames, vm.Name) + } + assert.Contains(t, volumeMountNames, "ceph-admin-keyring") + assert.Contains(t, volumeMountNames, "gateway-config") + assert.Contains(t, volumeMountNames, "ceph-conf-emptydir") + }) + + t.Run("daemon container configuration", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + } + + r, configHash := newDeploymentSpecTest(t) + + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + + assert.Len(t, d.Spec.Template.Spec.Containers, 1) + daemonCont := d.Spec.Template.Spec.Containers[0] + assert.Equal(t, "nvmeof-gateway", daemonCont.Name) + assert.Equal(t, nvmeof.Spec.Image, daemonCont.Image) + + expectedCephArgs := strings.Join(cephconfig.DefaultFlags(r.clusterInfo.FSID, "/etc/ceph/keyring"), " ") + assertEnvVar(t, daemonCont.Env, "CEPH_ARGS", expectedCephArgs) + + portNames := []string{} + for _, p := range daemonCont.Ports { + portNames = append(portNames, p.Name) + } + assert.Contains(t, portNames, "io") + assert.Contains(t, portNames, "gateway") + assert.Contains(t, portNames, "monitor") + assert.Contains(t, portNames, "discovery") + + assert.NotNil(t, daemonCont.SecurityContext) + // SecurityContext should be set with privileged=true for NVMeOF gateway + assert.NotNil(t, daemonCont.SecurityContext.Privileged) + assert.True(t, *daemonCont.SecurityContext.Privileged) + }) + + t.Run("daemon container uses default image when not specified", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + // Image not specified - should use default + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + } + + r, configHash := newDeploymentSpecTest(t) + + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + + assert.Len(t, d.Spec.Template.Spec.Containers, 1) + daemonCont := d.Spec.Template.Spec.Containers[0] + assert.Equal(t, "nvmeof-gateway", daemonCont.Name) + assert.Equal(t, "quay.io/ceph/nvmeof:1.5", daemonCont.Image) + }) + + t.Run("hostname set when instance name is valid", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + } + + r, configHash := newDeploymentSpecTest(t) + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + assert.Equal(t, instanceName(nvmeof, "0"), d.Spec.Template.Spec.Hostname) + }) + + t.Run("hostname not set when instance name is invalid", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my_nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + } + + r, configHash := newDeploymentSpecTest(t) + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + assert.Empty(t, d.Spec.Template.Spec.Hostname) + }) + + t.Run("service generation", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + }, + } + + r, _ := newDeploymentSpecTest(t) + + svc := r.generateCephNVMeOFService(nvmeof, "0") + assert.Equal(t, "rook-ceph-nvmeof-my-nvmeof-0", svc.Name) + assert.Equal(t, "rook-ceph-test-ns", svc.Namespace) + + // Verify service labels contain Ceph requirements + daemonID := nvmeof.Name + "-0" + cephTest.AssertLabelsContainCephRequirements(t, svc.Labels, + "nvmeof", daemonID, AppName, nvmeof.Namespace, nvmeof.Name, + "cephnvmeofgateways.ceph.rook.io", "ceph-nvmeof") + + portNames := []string{} + for _, p := range svc.Spec.Ports { + portNames = append(portNames, p.Name) + } + assert.Contains(t, portNames, "io") + assert.Contains(t, portNames, "gateway") + assert.Contains(t, portNames, "monitor") + assert.Contains(t, portNames, "discovery") + + for _, p := range svc.Spec.Ports { + switch p.Name { + case "io": + assert.Equal(t, int32(nvmeofIOPort), p.Port) + case "gateway": + assert.Equal(t, int32(nvmeofGatewayPort), p.Port) + case "monitor": + assert.Equal(t, int32(nvmeofMonitorPort), p.Port) + case "discovery": + assert.Equal(t, int32(nvmeofDiscoveryPort), p.Port) + } + } + }) + + t.Run("custom ports", func(t *testing.T) { + nvmeof := &cephv1.CephNVMeOFGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nvmeof", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NVMeOFGatewaySpec{ + Image: "quay.io/ceph/nvmeof:1.5", + Pool: "nvmeof", + Group: "group-a", + Instances: 1, + Ports: &cephv1.NVMeOFGatewayPorts{ + IOPort: 14420, + GatewayPort: 15500, + MonitorPort: 15499, + DiscoveryPort: 18009, + }, + }, + } + + r, configHash := newDeploymentSpecTest(t) + + configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name) + d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash) + assert.NoError(t, err) + + daemonCont := d.Spec.Template.Spec.Containers[0] + assertContainerPort(t, daemonCont.Ports, "io", 14420) + assertContainerPort(t, daemonCont.Ports, "gateway", 15500) + assertContainerPort(t, daemonCont.Ports, "monitor", 15499) + assertContainerPort(t, daemonCont.Ports, "discovery", 18009) + + svc := r.generateCephNVMeOFService(nvmeof, "0") + assertServicePort(t, svc.Spec.Ports, "io", 14420) + assertServicePort(t, svc.Spec.Ports, "gateway", 15500) + assertServicePort(t, svc.Spec.Ports, "monitor", 15499) + assertServicePort(t, svc.Spec.Ports, "discovery", 18009) + }) +} + +func assertEnvVar(t *testing.T, envVars []v1.EnvVar, name, value string) { + t.Helper() + for _, env := range envVars { + if env.Name == name { + assert.Equal(t, value, env.Value) + return + } + } + assert.Failf(t, "missing env var", "expected %s", name) +} + +func assertEnvVarPresent(t *testing.T, envVars []v1.EnvVar, name string) { + t.Helper() + for _, env := range envVars { + if env.Name == name { + assert.NotNil(t, env.ValueFrom) + return + } + } + assert.Failf(t, "missing env var", "expected %s", name) +} + +func assertServicePort(t *testing.T, ports []v1.ServicePort, name string, value int32) { + t.Helper() + for _, port := range ports { + if port.Name == name { + assert.Equal(t, value, port.Port) + return + } + } + assert.Failf(t, "missing port", "expected %s", name) +} + +func assertContainerPort(t *testing.T, ports []v1.ContainerPort, name string, value int32) { + t.Helper() + for _, port := range ports { + if port.Name == name { + assert.Equal(t, value, port.ContainerPort) + return + } + } + assert.Failf(t, "missing port", "expected %s", name) +}