Labels are label for a given daemons
@@ -8815,7 +9053,7 @@ int
LocalCephxStatus
-(Appears on:CephFilesystemStatus, FileMirrorStatus, NFSStatus, ObjectStoreStatus, RBDMirrorStatus)
+(Appears on:CephFilesystemStatus, FileMirrorStatus, NFSStatus, NVMeOFGatewayStatus, ObjectStoreStatus, RBDMirrorStatus)
@@ -10207,6 +10445,315 @@ LocalCephxStatus
+
NVMeOFGatewayPorts
+
+
+(Appears on:NVMeOFGatewaySpec)
+
+
+
NVMeOFGatewayPorts represents the port configuration for NVMe-oF gateway
+
+
+
+
+| Field |
+Description |
+
+
+
+
+
+ioPort
+
+int32
+
+ |
+
+(Optional)
+ IOPort is the port for NVMe-oF IO traffic (default: 4420)
+ |
+
+
+
+gatewayPort
+
+int32
+
+ |
+
+(Optional)
+ GatewayPort is the port for the gateway service (default: 5500)
+ |
+
+
+
+monitorPort
+
+int32
+
+ |
+
+(Optional)
+ MonitorPort is the port for the monitor service (default: 5499)
+ |
+
+
+
+discoveryPort
+
+int32
+
+ |
+
+(Optional)
+ DiscoveryPort is the port for discovery service (default: 8009)
+ |
+
+
+
+
NVMeOFGatewaySpec
+
+
+(Appears on:CephNVMeOFGateway)
+
+
+
NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway
+
+
+
+
+| Field |
+Description |
+
+
+
+
+
+image
+
+string
+
+ |
+
+ Image is the container image to use for the NVMe-oF gateway daemon.
+For example, quay.io/ceph/nvmeof:1.5
+ |
+
+
+
+instances
+
+int
+
+ |
+
+ The number of active gateway instances
+ |
+
+
+
+pool
+
+string
+
+ |
+
+ Pool is the RADOS pool where NVMe-oF configuration is stored
+ |
+
+
+
+group
+
+string
+
+ |
+
+ Group is the gateway group name for high availability (ANA group)
+ |
+
+
+
+configMapRef
+
+string
+
+ |
+
+(Optional)
+ ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration
+If not specified, a default configuration will be generated
+ |
+
+
+
+nvmeofConfig
+
+map[string]map[string]string
+
+ |
+
+(Optional)
+ NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration
+This allows users to override or add configuration options without needing to manage a ConfigMap
+ |
+
+
+
+placement
+
+
+Placement
+
+
+ |
+
+(Optional)
+ The affinity to place the gateway pods
+ |
+
+
+
+annotations
+
+
+Annotations
+
+
+ |
+
+(Optional)
+ The annotations-related configuration to add/set on each Pod related object.
+ |
+
+
+
+labels
+
+
+Labels
+
+
+ |
+
+(Optional)
+ The labels-related configuration to add/set on each Pod related object.
+ |
+
+
+
+resources
+
+
+Kubernetes core/v1.ResourceRequirements
+
+
+ |
+
+(Optional)
+ Resources set resource requests and limits
+ |
+
+
+
+priorityClassName
+
+string
+
+ |
+
+(Optional)
+ PriorityClassName sets the priority class on the pods
+ |
+
+
+
+hostNetwork
+
+bool
+
+ |
+
+(Optional)
+ Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.
+ |
+
+
+
+ports
+
+
+NVMeOFGatewayPorts
+
+
+ |
+
+(Optional)
+ Ports configuration for the NVMe-oF gateway
+ |
+
+
+
+livenessProbe
+
+
+ProbeSpec
+
+
+ |
+
+(Optional)
+ A liveness-probe to verify that gateway has valid run-time state.
+If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
+ |
+
+
+
+
NVMeOFGatewayStatus
+
+
+(Appears on:CephNVMeOFGateway)
+
+
+
NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway
+
+
+
+
+| Field |
+Description |
+
+
+
+
+
+Status
+
+
+Status
+
+
+ |
+
+
+(Members of Status are embedded into this type.)
+
+ |
+
+
+
+cephx
+
+
+LocalCephxStatus
+
+
+ |
+
+ |
+
+
+
NamedBlockPoolSpec
@@ -12299,7 +12846,7 @@ string
Placement
-(Appears on:CephCOSIDriverSpec, FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, RBDMirroringSpec, StorageClassDeviceSet)
+(Appears on:CephCOSIDriverSpec, FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, NVMeOFGatewaySpec, RBDMirroringSpec, StorageClassDeviceSet)
Placement is the placement for an object
@@ -12722,7 +13269,7 @@ string
ProbeSpec
-(Appears on:GaneshaServerSpec, MetadataServerSpec, ObjectHealthCheckSpec)
+(Appears on:GaneshaServerSpec, MetadataServerSpec, NVMeOFGatewaySpec, ObjectHealthCheckSpec)
ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon
@@ -14216,7 +14763,7 @@ int
Status
-(Appears on:CephBucketNotification, CephObjectRealm, CephObjectZone, CephObjectZoneGroup, FileMirrorStatus, NFSStatus, RBDMirrorStatus)
+(Appears on:CephBucketNotification, CephObjectRealm, CephObjectZone, CephObjectZoneGroup, FileMirrorStatus, NFSStatus, NVMeOFGatewayStatus, RBDMirrorStatus)
Status represents the status of an object
diff --git a/Documentation/Storage-Configuration/Block-Storage-RBD/nvme-of.md b/Documentation/Storage-Configuration/Block-Storage-RBD/nvme-of.md
new file mode 100644
index 000000000..3d4c70780
--- /dev/null
+++ b/Documentation/Storage-Configuration/Block-Storage-RBD/nvme-of.md
@@ -0,0 +1,462 @@
+---
+title: NVMe-oF Block Storage
+---
+# NVMe-oF Block Storage
+
+**This feature is experimental**
+
+NVMe over Fabrics (NVMe-oF) allows RBD volumes to be exposed and accessed via the NVMe/TCP protocol. This enables both Kubernetes pods within the cluster and external clients outside the cluster to connect to Ceph block storage using standard NVMe-oF initiators, providing high-performance block storage access over the network.
+
+## Goals
+
+The NVMe-oF integration in Rook serves two primary purposes:
+
+1. **External Client Access**: Rook serves as a backend for external clients outside the cluster, enabling non-Kubernetes workloads to access Ceph block storage through standard NVMe-oF initiators. This allows organizations to leverage their Ceph storage infrastructure for both containerized and traditional workloads.
+
+2. **In-Cluster Consumption**: Pods inside the Kubernetes cluster can consume storage via the NVMe-oF protocol, providing an alternative to traditional RBD mounts with potential performance benefits for certain workloads.
+
+Both use cases are supported, allowing you to choose the appropriate access method based on your specific requirements and deployment scenarios.
+
+For more background and design details, see the [NVMe-oF gateway design doc](https://github.com/rook/rook/blob/master/design/ceph/ceph-nvmeof-gateway.md).
+For the Ceph-CSI NVMe-oF design proposal, see the [ceph-csi NVMe-oF proposal](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/nvme-of.md).
+
+## Prerequisites
+
+This guide assumes a Rook cluster as explained in the [Quickstart Guide](../../Getting-Started/quickstart.md).
+
+### Requirements
+
+- **Ceph Version**: Ceph v20 (Tentacle) or later
+- **Disable the Ceph CSI operator**: We are still updating the Ceph CSI operator with NVMe-oF support. Currently, it is required to disable the CSI operator to test NVMe-oF. In operator.yaml, set `ROOK_USE_CSI_OPERATOR: "false"`.
+
+## Step 1: Create a Ceph Block Pool
+
+Before creating the NVMe-oF gateway, you need to create a CephBlockPool that will be used by the gateway:
+
+```yaml
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: nvmeof
+ namespace: rook-ceph
+spec:
+ failureDomain: host
+ replicated:
+ size: 3
+```
+
+Create the pool:
+
+```console
+kubectl create -f deploy/examples/csi/nvmeof/nvmeof-pool.yaml
+```
+
+## Step 2: Create the NVMe-oF Gateway
+
+The `CephNVMeOFGateway` CRD manages the NVMe-oF gateway infrastructure. The operator will automatically create the following resources:
+
+- **Service**: One per gateway instance for service discovery
+- **Deployment**: One per gateway instance running the NVMe-oF gateway daemon
+
+Create the gateway:
+
+```yaml
+apiVersion: ceph.rook.io/v1
+kind: CephNVMeOFGateway
+metadata:
+ name: nvmeof
+ namespace: rook-ceph
+spec:
+ # Container image for the NVMe-oF gateway daemon
+ image: quay.io/ceph/nvmeof:1.5
+ # Pool name that will be used by the NVMe-oF gateway
+ pool: nvmeof
+ # ANA (Asymmetric Namespace Access) group name
+ group: group-a
+ # Number of gateway instances to run
+ instances: 1
+ hostNetwork: false
+```
+
+Apply the gateway configuration:
+
+```console
+kubectl create -f deploy/examples/nvmeof-test.yaml
+```
+
+Verify the gateway is running:
+
+```console
+kubectl get pod -n rook-ceph -l app=rook-ceph-nvmeof
+```
+
+**Example Output**
+
+```console
+NAME READY STATUS RESTARTS AGE
+rook-ceph-nvmeof-nvmeof-a-85844ff6b8-4r8gj 1/1 Running 0 91s
+```
+
+## Step 4: Deploy the NVMe-oF CSI Driver
+
+The NVMe-oF CSI driver handles dynamic provisioning of volumes. Deploy the CSI provisioner with the NVMe-oF driver.
+
+Deploy the NVMe-oF CSI provisioner from the example manifest:
+
+```console
+kubectl create -f deploy/examples/csi/nvmeof/provisioner.yaml
+```
+
+Verify the CSI provisioner pod is running:
+
+```console
+kubectl get pods -n rook-ceph -l app=csi-nvmeofplugin-provisioner
+```
+
+**Example Output**
+
+```console
+NAME READY STATUS RESTARTS AGE
+csi-nvmeofplugin-provisioner-65b4fbbc8-jjsqj 4/4 Running 0 75s
+```
+
+## Step 5: Create the StorageClass
+
+Create a StorageClass that uses the NVMe-oF CSI driver. You'll need to gather the following information from the gateway:
+
+1. **nvmeofGatewayAddress**: A stable address for the gateway management API
+2. **nvmeofGatewayPort**: The gateway port (default: 5500)
+3. **listeners**: A JSON array containing listener information for each gateway instance
+
+Discover the values to use in the StorageClass:
+
+1. **nvmeofGatewayAddress**: Use the Service `CLUSTER-IP`.
+
+ ```console
+ kubectl get service -n rook-ceph rook-ceph-nvmeof-nvmeof-a
+ ```
+
+ **Example Output**
+
+ ```console
+ NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
+ rook-ceph-nvmeof-nvmeof-a ClusterIP 10.106.98.71
4420/TCP,5500/TCP,5499/TCP,8009/TCP 24m
+ ```
+
+2. **listeners.address**: Use the gateway pod IP.
+
+ ```console
+ kubectl get pods -n rook-ceph -l app=rook-ceph-nvmeof -o wide
+
+ ```
+
+ **Example Output**
+
+ ```console
+ NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
+ rook-ceph-nvmeof-nvmeof-a-5fd6cd4d46-mrbwk 1/1 Running 0 26m 10.244.0.16 minikube
+ ```
+
+3. **listeners.hostname**: Use the gateway deployment name.
+
+ ```console
+ kubectl get deployments.apps -n rook-ceph -l app=rook-ceph-nvmeof
+ ```
+
+ **Example Output**
+
+ ```console
+ NAME READY UP-TO-DATE AVAILABLE AGE
+ rook-ceph-nvmeof-nvmeof-a 1/1 1 1 27m
+ ```
+
+Create the StorageClass:
+
+
+```yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: ceph-nvmeof
+parameters:
+ clusterID: rook-ceph
+ pool: nvmeof
+ subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph
+ # Management API - talks to gateway to create subsystems/namespaces
+ nvmeofGatewayAddress: "10.106.98.71"
+ nvmeofGatewayPort: "5500"
+ # Data Plane - worker nodes connect here for actual I/O
+ # List ALL gateway pods for HA and multipath
+ listeners: |
+ [
+ {
+ "address": "10.244.0.16",
+ "port": 4420,
+ "hostname": "rook-ceph-nvmeof-nvmeof-a"
+ }
+ ]
+ csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
+ csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+ csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
+ csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+ imageFormat: "2"
+ imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff
+provisioner: nvmeof.csi.ceph.com
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: false
+```
+
+Create the StorageClass:
+
+```console
+kubectl create -f deploy/examples/csi/nvmeof/storageclass.yaml
+```
+
+## Step 6: Create a PersistentVolumeClaim
+
+Create a PVC using the NVMe-oF storage class:
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: nvmeof-external-volume
+ namespace: default
+spec:
+ storageClassName: ceph-nvmeof
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 128Mi
+```
+
+!!! note
+ This PVC is created for CSI driver provisioning. The volume will be accessible via NVMe-oF protocol by both Kubernetes pods within the cluster and external clients outside the cluster using standard NVMe-oF initiators.
+
+Create the PVC:
+
+```console
+kubectl create -f deploy/examples/csi/nvmeof/pvc.yaml
+```
+
+Verify the PVC is bound:
+
+```console
+kubectl get pvc nvmeof-external-volume
+```
+
+**Example Output**
+
+```console
+NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
+nvmeof-external-volume Bound pvc-b4108580-5cfa-46d3-beff-320088a5bf3c 128Mi RWO ceph-nvmeof 20m
+```
+
+## Step 7: Deploy the NVMe-oF CSI Node Plugin
+
+Deploy the NVMe-oF CSI node plugin:
+
+```console
+kubectl create -f deploy/examples/csi/nvmeof/node-plugin.yaml
+```
+
+Verify the node plugin pod is running:
+
+```console
+kubectl get pods -n rook-ceph -l app=nvmeof.csi.ceph.com-nodeplugin
+```
+
+**Example Output**
+
+```console
+NAME READY STATUS RESTARTS AGE
+nvmeof.csi.ceph.com-nodeplugin-xnm82 2/2 Running 0 31h
+```
+
+## Step 8: Accessing Volumes via NVMe-oF
+
+Once the PVC is created and bound, the volume is available via NVMe-oF. The volume can be accessed by both Kubernetes pods within the cluster and external clients outside the cluster.
+
+### Access from Kubernetes Pods
+
+Kubernetes pods can consume NVMe-oF volumes by mounting the PVC directly. The CSI driver handles the NVMe-oF connection automatically when the pod mounts the volume.
+
+Create a sample pod that mounts the PVC:
+
+```console
+kubectl create -f deploy/examples/csi/nvmeof/pod.yaml
+```
+
+Verify the pod is running:
+
+```console
+kubectl get pods -n default
+```
+
+**Example Output**
+
+```console
+NAME READY STATUS RESTARTS AGE
+nvmeof-test-pod 1/1 Running 0 29h
+```
+
+### Access from External Clients
+
+External clients outside the Kubernetes cluster can connect to the gateway using standard NVMe-oF procedures.
+
+#### Prerequisites for External Clients
+
+- **NVMe-oF Initiator**: The client must have the `nvme-tcp` kernel module loaded and `nvme-cli` installed
+- **Network Access**: The client must be able to reach the gateway service IP and ports
+
+#### Discover Subsystems
+
+From the external client, discover available NVMe-oF subsystems:
+
+```bash
+nvme discover -t tcp -a -s 5500
+```
+
+Replace `` with the gateway service ClusterIP or an accessible endpoint.
+
+#### Connect to Subsystem
+
+Connect to the discovered subsystem:
+
+```bash
+nvme connect -t tcp -n -a -s 5500
+```
+
+Replace:
+
+- `` with the `subsystemNQN` value from your StorageClass (e.g., `nqn.2016-06.io.spdk:cnode1.rook-ceph`)
+- `` with the gateway service IP or pod IP
+
+#### Access the Volume
+
+Once connected, the NVMe namespace will appear as a block device on the client:
+
+```bash
+lsblk | grep nvme
+```
+
+The device will typically appear as `/dev/nvmeXnY` where X is the controller number and Y is the namespace ID.
+
+#### Format and Mount (Optional)
+
+If you want to format and mount the device:
+
+```bash
+# Format the device
+sudo mkfs.ext4 /dev/nvmeXnY
+
+# Mount the device
+sudo mkdir /mnt/nvmeof
+sudo mount /dev/nvmeXnY /mnt/nvmeof
+```
+
+## High Availability
+
+For production deployments, configure multiple gateway instances for high availability:
+
+1. **Increase Gateway Instances**: Set `instances: 2` or higher in the `CephNVMeOFGateway` spec
+2. **Update StorageClass Listeners**: Add all gateway instance addresses and instance names to the `listeners` array
+3. **Load Balancing**: Each gateway instance has its own Service; list all of them to support multipath/HA
+
+Example with multiple instances:
+
+```yaml
+spec:
+ instances: 2
+ # ... other settings
+```
+
+Then update the StorageClass `listeners` to include all gateway instances/services:
+
+```yaml
+listeners: |
+ [
+ {
+ "address": "10.99.212.218",
+ "port": 4420,
+ "hostname": "rook-ceph-nvmeof-nvmeof-a"
+ },
+ {
+ "address": "10.99.212.219",
+ "port": 4420,
+ "hostname": "rook-ceph-nvmeof-nvmeof-b"
+ }
+ ]
+```
+
+## Troubleshooting
+
+### Check Gateway Pod Logs
+
+```console
+kubectl logs -n rook-ceph -l app=rook-ceph-nvmeof --tail=100
+```
+
+### Check CSI Provisioner Logs
+
+```console
+kubectl logs -n rook-ceph -l app=csi-nvmeofplugin-provisioner --tail=100
+```
+
+### Verify Gateway Service
+
+```console
+kubectl describe service -n rook-ceph rook-ceph-nvmeof-my-nvmeof-0
+```
+
+### Check PVC Events
+
+```console
+kubectl describe pvc nvmeof-external-volume
+```
+
+### Verify Ceph CSI Config
+
+Ensure the `rook-ceph-csi-config` ConfigMap exists and contains the cluster configuration:
+
+```console
+kubectl get configmap -n rook-ceph rook-ceph-csi-config -o yaml
+```
+
+## Teardown
+
+!!! warning
+ Deleting the PVC will also delete the underlying RBD image and NVMe namespace. Ensure you have backups if needed.
+
+To clean up all the artifacts created:
+
+```console
+# Delete the test pod
+kubectl delete -f deploy/examples/csi/nvmeof/pod.yaml
+
+# Delete the PVC
+kubectl delete pvc nvmeof-external-volume
+
+# Delete the StorageClass
+kubectl delete storageclass ceph-nvmeof
+
+# Delete the NVMe-oF CSI node plugin
+kubectl delete -f deploy/examples/csi/nvmeof/node-plugin.yaml
+
+# Delete the NVMe-oF CSI provisioner (includes ServiceAccount/RBAC)
+kubectl delete -f deploy/examples/csi/nvmeof/provisioner.yaml
+
+# Delete the NVMe-oF gateway
+kubectl delete -n rook-ceph cephnvmeofgateway.ceph.rook.io my-nvmeof
+
+# Delete the block pool (optional)
+kubectl delete -n rook-ceph cephblockpool.ceph.rook.io nvmeof
+```
+
+## References
+
+- [Ceph NVMe-oF Documentation](https://docs.ceph.com/en/latest/rbd/nvmeof-overview/)
+- [Ceph NVMe-oF Initiator Guide](https://docs.ceph.com/en/latest/rbd/nvmeof-initiator-linux/)
+- [Ceph CSI NVMe-oF Support](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/nvme-of.md)
diff --git a/deploy/charts/library/templates/_cluster-role.tpl b/deploy/charts/library/templates/_cluster-role.tpl
index a95736641..00cf8c250 100644
--- a/deploy/charts/library/templates/_cluster-role.tpl
+++ b/deploy/charts/library/templates/_cluster-role.tpl
@@ -65,6 +65,7 @@ rules:
- cephblockpools
- cephfilesystems
- cephnfses
+ - cephnvmeofgateways
- cephobjectstores
- cephobjectstoreusers
- cephobjectrealms
diff --git a/deploy/charts/library/templates/_cluster-serviceaccount.tpl b/deploy/charts/library/templates/_cluster-serviceaccount.tpl
index 02bd17f87..1b79a46c7 100644
--- a/deploy/charts/library/templates/_cluster-serviceaccount.tpl
+++ b/deploy/charts/library/templates/_cluster-serviceaccount.tpl
@@ -62,4 +62,14 @@ metadata:
labels:
{{- include "library.rook-ceph.labels" . | nindent 4 }}
{{- include "library.imagePullSecrets" . | nindent 0 }}
+---
+# Service account for NVMe-oF gateway
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+ name: rook-ceph-nvmeof
+ namespace: {{ .Release.Namespace }} # namespace:cluster
+ labels:
+ {{- include "library.rook-ceph.labels" . | nindent 4 }}
+{{- include "library.imagePullSecrets" . | nindent 0 }}
{{- end }}
diff --git a/deploy/charts/rook-ceph/templates/clusterrole.yaml b/deploy/charts/rook-ceph/templates/clusterrole.yaml
index d1bf92fc9..4f1f0f4e0 100644
--- a/deploy/charts/rook-ceph/templates/clusterrole.yaml
+++ b/deploy/charts/rook-ceph/templates/clusterrole.yaml
@@ -143,6 +143,7 @@ rules:
- cephblockpools
- cephfilesystems
- cephnfses
+ - cephnvmeofgateways
- cephobjectstores
- cephobjectstoreusers
- cephobjectrealms
@@ -169,6 +170,7 @@ rules:
- cephblockpools/status
- cephfilesystems/status
- cephnfses/status
+ - cephnvmeofgateways/status
- cephobjectstores/status
- cephobjectstoreusers/status
- cephobjectrealms/status
@@ -192,6 +194,7 @@ rules:
- cephblockpools/finalizers
- cephfilesystems/finalizers
- cephnfses/finalizers
+ - cephnvmeofgateways/finalizers
- cephobjectstores/finalizers
- cephobjectstoreusers/finalizers
- cephobjectrealms/finalizers
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index de5f17437..50ab6361d 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -11510,6 +11510,920 @@ spec:
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
+metadata:
+ annotations:
+ controller-gen.kubebuilder.io/version: v0.19.0
+ helm.sh/resource-policy: keep
+ name: cephnvmeofgateways.ceph.rook.io
+spec:
+ group: ceph.rook.io
+ names:
+ kind: CephNVMeOFGateway
+ listKind: CephNVMeOFGatewayList
+ plural: cephnvmeofgateways
+ shortNames:
+ - nvmeof
+ singular: cephnvmeofgateway
+ scope: Namespaced
+ versions:
+ - additionalPrinterColumns:
+ - jsonPath: .status.phase
+ name: Phase
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ name: Age
+ type: date
+ name: v1
+ schema:
+ openAPIV3Schema:
+ description: CephNVMeOFGateway represents a Ceph NVMe-oF Gateway
+ properties:
+ apiVersion:
+ description: |-
+ APIVersion defines the versioned schema of this representation of an object.
+ Servers should convert recognized schemas to the latest internal value, and
+ may reject unrecognized values.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+ type: string
+ kind:
+ description: |-
+ Kind is a string value representing the REST resource this object represents.
+ Servers may infer this from the endpoint the client submits requests to.
+ Cannot be updated.
+ In CamelCase.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway
+ properties:
+ annotations:
+ additionalProperties:
+ type: string
+ description: The annotations-related configuration to add/set on each Pod related object.
+ type: object
+ configMapRef:
+ description: |-
+ ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration
+ If not specified, a default configuration will be generated
+ minLength: 1
+ type: string
+ group:
+ description: Group is the gateway group name for high availability (ANA group)
+ minLength: 1
+ type: string
+ hostNetwork:
+ description: Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.
+ type: boolean
+ image:
+ description: |-
+ Image is the container image to use for the NVMe-oF gateway daemon.
+ For example, quay.io/ceph/nvmeof:1.5
+ minLength: 1
+ type: string
+ instances:
+ description: The number of active gateway instances
+ minimum: 1
+ type: integer
+ labels:
+ additionalProperties:
+ type: string
+ description: The labels-related configuration to add/set on each Pod related object.
+ type: object
+ livenessProbe:
+ description: |-
+ A liveness-probe to verify that gateway has valid run-time state.
+ If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
+ properties:
+ disabled:
+ description: Disabled determines whether probe is disable or not
+ type: boolean
+ probe:
+ description: |-
+ Probe describes a health check to be performed against a container to determine whether it is
+ alive or ready to receive traffic.
+ properties:
+ exec:
+ description: Exec specifies a command to execute in the container.
+ properties:
+ command:
+ description: |-
+ Command is the command line to execute inside the container, the working directory for the
+ command is root ('/') in the container's filesystem. The command is simply exec'd, it is
+ not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use
+ a shell, you need to explicitly call out to that shell.
+ Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ failureThreshold:
+ description: |-
+ Minimum consecutive failures for the probe to be considered failed after having succeeded.
+ Defaults to 3. Minimum value is 1.
+ format: int32
+ type: integer
+ grpc:
+ description: GRPC specifies a GRPC HealthCheckRequest.
+ properties:
+ port:
+ description: Port number of the gRPC service. Number must be in the range 1 to 65535.
+ format: int32
+ type: integer
+ service:
+ default: ""
+ description: |-
+ Service is the name of the service to place in the gRPC HealthCheckRequest
+ (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).
+
+ If this is not specified, the default behavior is defined by gRPC.
+ type: string
+ required:
+ - port
+ type: object
+ httpGet:
+ description: HTTPGet specifies an HTTP GET request to perform.
+ properties:
+ host:
+ description: |-
+ Host name to connect to, defaults to the pod IP. You probably want to set
+ "Host" in httpHeaders instead.
+ type: string
+ httpHeaders:
+ description: Custom headers to set in the request. HTTP allows repeated headers.
+ items:
+ description: HTTPHeader describes a custom header to be used in HTTP probes
+ properties:
+ name:
+ description: |-
+ The header field name.
+ This will be canonicalized upon output, so case-variant names will be understood as the same header.
+ type: string
+ value:
+ description: The header field value
+ type: string
+ required:
+ - name
+ - value
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ path:
+ description: Path to access on the HTTP server.
+ type: string
+ port:
+ anyOf:
+ - type: integer
+ - type: string
+ description: |-
+ Name or number of the port to access on the container.
+ Number must be in the range 1 to 65535.
+ Name must be an IANA_SVC_NAME.
+ x-kubernetes-int-or-string: true
+ scheme:
+ description: |-
+ Scheme to use for connecting to the host.
+ Defaults to HTTP.
+ type: string
+ required:
+ - port
+ type: object
+ initialDelaySeconds:
+ description: |-
+ Number of seconds after the container has started before liveness probes are initiated.
+ More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
+ format: int32
+ type: integer
+ periodSeconds:
+ description: |-
+ How often (in seconds) to perform the probe.
+ Default to 10 seconds. Minimum value is 1.
+ format: int32
+ type: integer
+ successThreshold:
+ description: |-
+ Minimum consecutive successes for the probe to be considered successful after having failed.
+ Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
+ format: int32
+ type: integer
+ tcpSocket:
+ description: TCPSocket specifies a connection to a TCP port.
+ properties:
+ host:
+ description: 'Optional: Host name to connect to, defaults to the pod IP.'
+ type: string
+ port:
+ anyOf:
+ - type: integer
+ - type: string
+ description: |-
+ Number or name of the port to access on the container.
+ Number must be in the range 1 to 65535.
+ Name must be an IANA_SVC_NAME.
+ x-kubernetes-int-or-string: true
+ required:
+ - port
+ type: object
+ terminationGracePeriodSeconds:
+ format: int64
+ type: integer
+ timeoutSeconds:
+ description: |-
+ Number of seconds after which the probe times out.
+ Defaults to 1 second. Minimum value is 1.
+ More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
+ format: int32
+ type: integer
+ type: object
+ type: object
+ nvmeofConfig:
+ additionalProperties:
+ additionalProperties:
+ type: string
+ type: object
+ description: |-
+ NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration
+ This allows users to override or add configuration options without needing to manage a ConfigMap
+ type: object
+ placement:
+ properties:
+ nodeAffinity:
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ preference:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchFields:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ x-kubernetes-map-type: atomic
+ weight:
+ format: int32
+ type: integer
+ required:
+ - preference
+ - weight
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ requiredDuringSchedulingIgnoredDuringExecution:
+ properties:
+ nodeSelectorTerms:
+ items:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchFields:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ x-kubernetes-map-type: atomic
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - nodeSelectorTerms
+ type: object
+ x-kubernetes-map-type: atomic
+ type: object
+ podAffinity:
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ podAffinityTerm:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ mismatchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ namespaceSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ topologyKey:
+ type: string
+ required:
+ - topologyKey
+ type: object
+ weight:
+ format: int32
+ type: integer
+ required:
+ - podAffinityTerm
+ - weight
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ requiredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ mismatchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ namespaceSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ topologyKey:
+ type: string
+ required:
+ - topologyKey
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ podAntiAffinity:
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ podAffinityTerm:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ mismatchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ namespaceSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ topologyKey:
+ type: string
+ required:
+ - topologyKey
+ type: object
+ weight:
+ format: int32
+ type: integer
+ required:
+ - podAffinityTerm
+ - weight
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ requiredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ mismatchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ namespaceSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ topologyKey:
+ type: string
+ required:
+ - topologyKey
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ tolerations:
+ items:
+ properties:
+ effect:
+ type: string
+ key:
+ type: string
+ operator:
+ type: string
+ tolerationSeconds:
+ format: int64
+ type: integer
+ value:
+ type: string
+ type: object
+ type: array
+ topologySpreadConstraints:
+ items:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ maxSkew:
+ format: int32
+ type: integer
+ minDomains:
+ format: int32
+ type: integer
+ nodeAffinityPolicy:
+ type: string
+ nodeTaintsPolicy:
+ type: string
+ topologyKey:
+ type: string
+ whenUnsatisfiable:
+ type: string
+ required:
+ - maxSkew
+ - topologyKey
+ - whenUnsatisfiable
+ type: object
+ type: array
+ type: object
+ pool:
+ description: Pool is the RADOS pool where NVMe-oF configuration is stored
+ minLength: 1
+ type: string
+ ports:
+ description: Ports configuration for the NVMe-oF gateway
+ properties:
+ discoveryPort:
+ description: 'DiscoveryPort is the port for discovery service (default: 8009)'
+ format: int32
+ maximum: 65535
+ minimum: 1
+ type: integer
+ gatewayPort:
+ description: 'GatewayPort is the port for the gateway service (default: 5500)'
+ format: int32
+ maximum: 65535
+ minimum: 1
+ type: integer
+ ioPort:
+ description: 'IOPort is the port for NVMe-oF IO traffic (default: 4420)'
+ format: int32
+ maximum: 65535
+ minimum: 1
+ type: integer
+ monitorPort:
+ description: 'MonitorPort is the port for the monitor service (default: 5499)'
+ format: int32
+ maximum: 65535
+ minimum: 1
+ type: integer
+ type: object
+ priorityClassName:
+ description: PriorityClassName sets the priority class on the pods
+ type: string
+ resources:
+ description: Resources set resource requests and limits
+ properties:
+ claims:
+ description: |-
+ Claims lists the names of resources, defined in spec.resourceClaims,
+ that are used by this container.
+
+ This field depends on the
+ DynamicResourceAllocation feature gate.
+
+ This field is immutable. It can only be set for containers.
+ items:
+ description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+ properties:
+ name:
+ description: |-
+ Name must match the name of one entry in pod.spec.resourceClaims of
+ the Pod where this field is used. It makes that resource available
+ inside a container.
+ type: string
+ request:
+ description: |-
+ Request is the name chosen for a request in the referenced claim.
+ If empty, everything from the claim is made available, otherwise
+ only the result of this request.
+ type: string
+ required:
+ - name
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - name
+ x-kubernetes-list-type: map
+ limits:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: |-
+ Limits describes the maximum amount of compute resources allowed.
+ More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+ type: object
+ requests:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: |-
+ Requests describes the minimum amount of compute resources required.
+ If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
+ otherwise to an implementation-defined value. Requests cannot exceed Limits.
+ More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+ type: object
+ type: object
+ required:
+ - group
+ - image
+ - instances
+ - pool
+ type: object
+ status:
+ description: NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway
+ properties:
+ cephx:
+ properties:
+ daemon:
+ description: Daemon shows the CephX key status for local Ceph daemons associated with this resources.
+ properties:
+ keyCephVersion:
+ description: |-
+ KeyCephVersion reports the Ceph version that created the current generation's keys. This is
+ same string format as reported by `CephCluster.status.version.version` to allow them to be
+ compared. E.g., `20.2.0-0`.
+ For all newly-created resources, this field set to the version of Ceph that created the key.
+ The special value "Uninitialized" indicates that keys are being created for the first time.
+ An empty string indicates that the version is unknown, as expected in brownfield deployments.
+ type: string
+ keyGeneration:
+ description: |-
+ KeyGeneration represents the CephX key generation for the last successful reconcile.
+ For all newly-created resources, this field is set to `1`.
+ When keys are rotated due to any rotation policy, the generation is incremented or updated to
+ the configured policy generation.
+ Generation `0` indicates that keys existed prior to the implementation of key tracking.
+ format: int32
+ type: integer
+ type: object
+ type: object
+ conditions:
+ items:
+ description: Condition represents a status condition on any Rook-Ceph Custom Resource.
+ properties:
+ lastHeartbeatTime:
+ format: date-time
+ type: string
+ lastTransitionTime:
+ format: date-time
+ type: string
+ message:
+ type: string
+ reason:
+ description: ConditionReason is a reason for a condition
+ type: string
+ status:
+ type: string
+ type:
+ description: ConditionType represent a resource's status
+ type: string
+ type: object
+ type: array
+ observedGeneration:
+ description: ObservedGeneration is the latest generation observed by the controller.
+ format: int64
+ type: integer
+ phase:
+ type: string
+ type: object
+ required:
+ - metadata
+ - spec
+ type: object
+ served: true
+ storage: true
+ subresources:
+ status: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.19.0
diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml
index 05b00aefc..e710ad413 100644
--- a/deploy/examples/common.yaml
+++ b/deploy/examples/common.yaml
@@ -364,6 +364,7 @@ rules:
- cephblockpools
- cephfilesystems
- cephnfses
+ - cephnvmeofgateways
- cephobjectstores
- cephobjectstoreusers
- cephobjectrealms
@@ -390,6 +391,7 @@ rules:
- cephblockpools/status
- cephfilesystems/status
- cephnfses/status
+ - cephnvmeofgateways/status
- cephobjectstores/status
- cephobjectstoreusers/status
- cephobjectrealms/status
@@ -413,6 +415,7 @@ rules:
- cephblockpools/finalizers
- cephfilesystems/finalizers
- cephnfses/finalizers
+ - cephnvmeofgateways/finalizers
- cephobjectstores/finalizers
- cephobjectstoreusers/finalizers
- cephobjectrealms/finalizers
@@ -1002,6 +1005,7 @@ rules:
- cephblockpools
- cephfilesystems
- cephnfses
+ - cephnvmeofgateways
- cephobjectstores
- cephobjectstoreusers
- cephobjectrealms
@@ -1400,6 +1404,20 @@ metadata:
# imagePullSecrets:
# - name: my-registry-secret
---
+# Service account for NVMe-oF gateway
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+ name: rook-ceph-nvmeof
+ namespace: rook-ceph # namespace:cluster
+ labels:
+ operator: rook
+ storage-backend: ceph
+ app.kubernetes.io/name: rook-ceph
+ app.kubernetes.io/part-of: rook-ceph-operator
+# imagePullSecrets:
+# - name: my-registry-secret
+---
# Service account for Ceph OSDs
kind: ServiceAccount
apiVersion: v1
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index d8fb21cb1..e94e7e0e1 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -11503,6 +11503,919 @@ spec:
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
+metadata:
+ annotations:
+ controller-gen.kubebuilder.io/version: v0.19.0
+ name: cephnvmeofgateways.ceph.rook.io
+spec:
+ group: ceph.rook.io
+ names:
+ kind: CephNVMeOFGateway
+ listKind: CephNVMeOFGatewayList
+ plural: cephnvmeofgateways
+ shortNames:
+ - nvmeof
+ singular: cephnvmeofgateway
+ scope: Namespaced
+ versions:
+ - additionalPrinterColumns:
+ - jsonPath: .status.phase
+ name: Phase
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ name: Age
+ type: date
+ name: v1
+ schema:
+ openAPIV3Schema:
+ description: CephNVMeOFGateway represents a Ceph NVMe-oF Gateway
+ properties:
+ apiVersion:
+ description: |-
+ APIVersion defines the versioned schema of this representation of an object.
+ Servers should convert recognized schemas to the latest internal value, and
+ may reject unrecognized values.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+ type: string
+ kind:
+ description: |-
+ Kind is a string value representing the REST resource this object represents.
+ Servers may infer this from the endpoint the client submits requests to.
+ Cannot be updated.
+ In CamelCase.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway
+ properties:
+ annotations:
+ additionalProperties:
+ type: string
+ description: The annotations-related configuration to add/set on each Pod related object.
+ type: object
+ configMapRef:
+ description: |-
+ ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration
+ If not specified, a default configuration will be generated
+ minLength: 1
+ type: string
+ group:
+ description: Group is the gateway group name for high availability (ANA group)
+ minLength: 1
+ type: string
+ hostNetwork:
+ description: Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.
+ type: boolean
+ image:
+ description: |-
+ Image is the container image to use for the NVMe-oF gateway daemon.
+ For example, quay.io/ceph/nvmeof:1.5
+ minLength: 1
+ type: string
+ instances:
+ description: The number of active gateway instances
+ minimum: 1
+ type: integer
+ labels:
+ additionalProperties:
+ type: string
+ description: The labels-related configuration to add/set on each Pod related object.
+ type: object
+ livenessProbe:
+ description: |-
+ A liveness-probe to verify that gateway has valid run-time state.
+ If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
+ properties:
+ disabled:
+ description: Disabled determines whether probe is disable or not
+ type: boolean
+ probe:
+ description: |-
+ Probe describes a health check to be performed against a container to determine whether it is
+ alive or ready to receive traffic.
+ properties:
+ exec:
+ description: Exec specifies a command to execute in the container.
+ properties:
+ command:
+ description: |-
+ Command is the command line to execute inside the container, the working directory for the
+ command is root ('/') in the container's filesystem. The command is simply exec'd, it is
+ not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use
+ a shell, you need to explicitly call out to that shell.
+ Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ failureThreshold:
+ description: |-
+ Minimum consecutive failures for the probe to be considered failed after having succeeded.
+ Defaults to 3. Minimum value is 1.
+ format: int32
+ type: integer
+ grpc:
+ description: GRPC specifies a GRPC HealthCheckRequest.
+ properties:
+ port:
+ description: Port number of the gRPC service. Number must be in the range 1 to 65535.
+ format: int32
+ type: integer
+ service:
+ default: ""
+ description: |-
+ Service is the name of the service to place in the gRPC HealthCheckRequest
+ (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).
+
+ If this is not specified, the default behavior is defined by gRPC.
+ type: string
+ required:
+ - port
+ type: object
+ httpGet:
+ description: HTTPGet specifies an HTTP GET request to perform.
+ properties:
+ host:
+ description: |-
+ Host name to connect to, defaults to the pod IP. You probably want to set
+ "Host" in httpHeaders instead.
+ type: string
+ httpHeaders:
+ description: Custom headers to set in the request. HTTP allows repeated headers.
+ items:
+ description: HTTPHeader describes a custom header to be used in HTTP probes
+ properties:
+ name:
+ description: |-
+ The header field name.
+ This will be canonicalized upon output, so case-variant names will be understood as the same header.
+ type: string
+ value:
+ description: The header field value
+ type: string
+ required:
+ - name
+ - value
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ path:
+ description: Path to access on the HTTP server.
+ type: string
+ port:
+ anyOf:
+ - type: integer
+ - type: string
+ description: |-
+ Name or number of the port to access on the container.
+ Number must be in the range 1 to 65535.
+ Name must be an IANA_SVC_NAME.
+ x-kubernetes-int-or-string: true
+ scheme:
+ description: |-
+ Scheme to use for connecting to the host.
+ Defaults to HTTP.
+ type: string
+ required:
+ - port
+ type: object
+ initialDelaySeconds:
+ description: |-
+ Number of seconds after the container has started before liveness probes are initiated.
+ More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
+ format: int32
+ type: integer
+ periodSeconds:
+ description: |-
+ How often (in seconds) to perform the probe.
+ Default to 10 seconds. Minimum value is 1.
+ format: int32
+ type: integer
+ successThreshold:
+ description: |-
+ Minimum consecutive successes for the probe to be considered successful after having failed.
+ Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
+ format: int32
+ type: integer
+ tcpSocket:
+ description: TCPSocket specifies a connection to a TCP port.
+ properties:
+ host:
+ description: 'Optional: Host name to connect to, defaults to the pod IP.'
+ type: string
+ port:
+ anyOf:
+ - type: integer
+ - type: string
+ description: |-
+ Number or name of the port to access on the container.
+ Number must be in the range 1 to 65535.
+ Name must be an IANA_SVC_NAME.
+ x-kubernetes-int-or-string: true
+ required:
+ - port
+ type: object
+ terminationGracePeriodSeconds:
+ format: int64
+ type: integer
+ timeoutSeconds:
+ description: |-
+ Number of seconds after which the probe times out.
+ Defaults to 1 second. Minimum value is 1.
+ More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
+ format: int32
+ type: integer
+ type: object
+ type: object
+ nvmeofConfig:
+ additionalProperties:
+ additionalProperties:
+ type: string
+ type: object
+ description: |-
+ NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration
+ This allows users to override or add configuration options without needing to manage a ConfigMap
+ type: object
+ placement:
+ properties:
+ nodeAffinity:
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ preference:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchFields:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ x-kubernetes-map-type: atomic
+ weight:
+ format: int32
+ type: integer
+ required:
+ - preference
+ - weight
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ requiredDuringSchedulingIgnoredDuringExecution:
+ properties:
+ nodeSelectorTerms:
+ items:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchFields:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ x-kubernetes-map-type: atomic
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - nodeSelectorTerms
+ type: object
+ x-kubernetes-map-type: atomic
+ type: object
+ podAffinity:
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ podAffinityTerm:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ mismatchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ namespaceSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ topologyKey:
+ type: string
+ required:
+ - topologyKey
+ type: object
+ weight:
+ format: int32
+ type: integer
+ required:
+ - podAffinityTerm
+ - weight
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ requiredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ mismatchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ namespaceSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ topologyKey:
+ type: string
+ required:
+ - topologyKey
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ podAntiAffinity:
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ podAffinityTerm:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ mismatchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ namespaceSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ topologyKey:
+ type: string
+ required:
+ - topologyKey
+ type: object
+ weight:
+ format: int32
+ type: integer
+ required:
+ - podAffinityTerm
+ - weight
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ requiredDuringSchedulingIgnoredDuringExecution:
+ items:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ mismatchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ namespaceSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ topologyKey:
+ type: string
+ required:
+ - topologyKey
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ type: object
+ tolerations:
+ items:
+ properties:
+ effect:
+ type: string
+ key:
+ type: string
+ operator:
+ type: string
+ tolerationSeconds:
+ format: int64
+ type: integer
+ value:
+ type: string
+ type: object
+ type: array
+ topologySpreadConstraints:
+ items:
+ properties:
+ labelSelector:
+ properties:
+ matchExpressions:
+ items:
+ properties:
+ key:
+ type: string
+ operator:
+ type: string
+ values:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ x-kubernetes-list-type: atomic
+ matchLabels:
+ additionalProperties:
+ type: string
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ maxSkew:
+ format: int32
+ type: integer
+ minDomains:
+ format: int32
+ type: integer
+ nodeAffinityPolicy:
+ type: string
+ nodeTaintsPolicy:
+ type: string
+ topologyKey:
+ type: string
+ whenUnsatisfiable:
+ type: string
+ required:
+ - maxSkew
+ - topologyKey
+ - whenUnsatisfiable
+ type: object
+ type: array
+ type: object
+ pool:
+ description: Pool is the RADOS pool where NVMe-oF configuration is stored
+ minLength: 1
+ type: string
+ ports:
+ description: Ports configuration for the NVMe-oF gateway
+ properties:
+ discoveryPort:
+ description: 'DiscoveryPort is the port for discovery service (default: 8009)'
+ format: int32
+ maximum: 65535
+ minimum: 1
+ type: integer
+ gatewayPort:
+ description: 'GatewayPort is the port for the gateway service (default: 5500)'
+ format: int32
+ maximum: 65535
+ minimum: 1
+ type: integer
+ ioPort:
+ description: 'IOPort is the port for NVMe-oF IO traffic (default: 4420)'
+ format: int32
+ maximum: 65535
+ minimum: 1
+ type: integer
+ monitorPort:
+ description: 'MonitorPort is the port for the monitor service (default: 5499)'
+ format: int32
+ maximum: 65535
+ minimum: 1
+ type: integer
+ type: object
+ priorityClassName:
+ description: PriorityClassName sets the priority class on the pods
+ type: string
+ resources:
+ description: Resources set resource requests and limits
+ properties:
+ claims:
+ description: |-
+ Claims lists the names of resources, defined in spec.resourceClaims,
+ that are used by this container.
+
+ This field depends on the
+ DynamicResourceAllocation feature gate.
+
+ This field is immutable. It can only be set for containers.
+ items:
+ description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+ properties:
+ name:
+ description: |-
+ Name must match the name of one entry in pod.spec.resourceClaims of
+ the Pod where this field is used. It makes that resource available
+ inside a container.
+ type: string
+ request:
+ description: |-
+ Request is the name chosen for a request in the referenced claim.
+ If empty, everything from the claim is made available, otherwise
+ only the result of this request.
+ type: string
+ required:
+ - name
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - name
+ x-kubernetes-list-type: map
+ limits:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: |-
+ Limits describes the maximum amount of compute resources allowed.
+ More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+ type: object
+ requests:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: |-
+ Requests describes the minimum amount of compute resources required.
+ If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
+ otherwise to an implementation-defined value. Requests cannot exceed Limits.
+ More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+ type: object
+ type: object
+ required:
+ - group
+ - image
+ - instances
+ - pool
+ type: object
+ status:
+ description: NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway
+ properties:
+ cephx:
+ properties:
+ daemon:
+ description: Daemon shows the CephX key status for local Ceph daemons associated with this resources.
+ properties:
+ keyCephVersion:
+ description: |-
+ KeyCephVersion reports the Ceph version that created the current generation's keys. This is
+ same string format as reported by `CephCluster.status.version.version` to allow them to be
+ compared. E.g., `20.2.0-0`.
+ For all newly-created resources, this field set to the version of Ceph that created the key.
+ The special value "Uninitialized" indicates that keys are being created for the first time.
+ An empty string indicates that the version is unknown, as expected in brownfield deployments.
+ type: string
+ keyGeneration:
+ description: |-
+ KeyGeneration represents the CephX key generation for the last successful reconcile.
+ For all newly-created resources, this field is set to `1`.
+ When keys are rotated due to any rotation policy, the generation is incremented or updated to
+ the configured policy generation.
+ Generation `0` indicates that keys existed prior to the implementation of key tracking.
+ format: int32
+ type: integer
+ type: object
+ type: object
+ conditions:
+ items:
+ description: Condition represents a status condition on any Rook-Ceph Custom Resource.
+ properties:
+ lastHeartbeatTime:
+ format: date-time
+ type: string
+ lastTransitionTime:
+ format: date-time
+ type: string
+ message:
+ type: string
+ reason:
+ description: ConditionReason is a reason for a condition
+ type: string
+ status:
+ type: string
+ type:
+ description: ConditionType represent a resource's status
+ type: string
+ type: object
+ type: array
+ observedGeneration:
+ description: ObservedGeneration is the latest generation observed by the controller.
+ format: int64
+ type: integer
+ phase:
+ type: string
+ type: object
+ required:
+ - metadata
+ - spec
+ type: object
+ served: true
+ storage: true
+ subresources:
+ status: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.19.0
diff --git a/deploy/examples/csi/nvmeof/node-plugin.yaml b/deploy/examples/csi/nvmeof/node-plugin.yaml
new file mode 100644
index 000000000..2319f04d8
--- /dev/null
+++ b/deploy/examples/csi/nvmeof/node-plugin.yaml
@@ -0,0 +1,202 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: nvmeof.csi.ceph.com-nodeplugin
+ namespace: rook-ceph
+spec:
+ revisionHistoryLimit: 10
+ selector:
+ matchLabels:
+ app: nvmeof.csi.ceph.com-nodeplugin
+ template:
+ metadata:
+ labels:
+ app: nvmeof.csi.ceph.com-nodeplugin
+ spec:
+ containers:
+ - args:
+ - --v=5
+ - --type=nvmeof
+ - --nodeserver=true
+ - --nodeid=$(NODE_ID)
+ - --drivername=nvmeof.csi.ceph.com
+ - --endpoint=unix:///csi/csi.sock
+ - --pidlimit=-1
+ - --csi-addons-endpoint=unix:///csi/csi-addons.sock
+ - --stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/
+ - --logtostderr=false
+ - --alsologtostderr=true
+ - --log_file=csi-logs/csi-nvmeofplugin.log
+ env:
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: status.podIP
+ - name: NODE_ID
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ image: quay.io/cephcsi/cephcsi:v3.16.0
+ imagePullPolicy: Always
+ name: csi-nvmeofplugin
+ resources: {}
+ securityContext:
+ allowPrivilegeEscalation: true
+ capabilities:
+ add:
+ - SYS_ADMIN
+ drop:
+ - All
+ privileged: true
+ terminationMessagePath: /dev/termination-log
+ terminationMessagePolicy: File
+ volumeMounts:
+ - mountPath: /dev
+ name: host-dev
+ - mountPath: /sys
+ name: host-sys
+ - mountPath: /run/mount
+ name: host-run-mount
+ - mountPath: /lib/modules
+ name: lib-modules
+ readOnly: true
+ - mountPath: /tmp/csi/keys
+ name: keys-tmp-dir
+ - mountPath: /csi
+ name: plugin-dir
+ # FIXED: Mount path should end with / to mount as directory
+ - mountPath: /etc/ceph-csi-config/
+ name: ceph-csi-config
+ - mountPath: /var/lib/kubelet/plugins
+ mountPropagation: Bidirectional
+ name: plugin-mount-dir
+ - mountPath: /var/lib/kubelet/pods
+ mountPropagation: Bidirectional
+ name: pods-mount-dir
+ - mountPath: /etc/selinux
+ name: etc-selinux
+ readOnly: true
+ - mountPath: /run/secrets/tokens
+ name: oidc-token
+ readOnly: true
+ - mountPath: /csi-logs
+ name: logs-dir
+ # Log rotation mount - commented out since ConfigMap is optional
+ # - mountPath: /etc/logrotate.d
+ # name: log-rotate-dir
+ # readOnly: true
+ - args:
+ - "--v=1"
+ - "--csi-address=/csi/csi.sock"
+ - "--kubelet-registration-path=/var/lib/kubelet/plugins/nvmeof.csi.ceph.com/csi.sock"
+ image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
+ imagePullPolicy: IfNotPresent
+ name: driver-registrar
+ resources: {}
+ securityContext:
+ capabilities:
+ drop:
+ - All
+ privileged: true
+ terminationMessagePath: /dev/termination-log
+ terminationMessagePolicy: File
+ volumeMounts:
+ - mountPath: /csi
+ name: plugin-dir
+ - mountPath: /registration
+ name: registration-dir
+ dnsPolicy: ClusterFirstWithHostNet
+ hostNetwork: true
+ hostPID: true
+ priorityClassName: system-node-critical
+ restartPolicy: Always
+ schedulerName: default-scheduler
+ securityContext: {}
+ # FIXED: Use rook-csi-rbd-plugin-sa since nvmeof uses RBD backend
+ # This service account already exists and has the necessary permissions
+ serviceAccount: rook-csi-rbd-plugin-sa
+ serviceAccountName: rook-csi-rbd-plugin-sa
+ terminationGracePeriodSeconds: 30
+ # tolerations:
+ # - effect: NoSchedule
+ # key: node.ocs.openshift.io/storage
+ # operator: Equal
+ # value: "true"
+ volumes:
+ - hostPath:
+ path: /dev
+ type: ""
+ name: host-dev
+ - hostPath:
+ path: /sys
+ type: ""
+ name: host-sys
+ - hostPath:
+ path: /run/mount
+ type: ""
+ name: host-run-mount
+ - hostPath:
+ path: /lib/modules
+ type: ""
+ name: lib-modules
+ - emptyDir:
+ medium: Memory
+ name: keys-tmp-dir
+ # FIXED: Use rook-ceph-csi-config instead of ceph-csi-config
+ # This ConfigMap contains the cluster configuration with monitor endpoints
+ - configMap:
+ name: rook-ceph-csi-config
+ items:
+ - key: csi-cluster-config-json
+ path: config.json
+ name: ceph-csi-config
+ - hostPath:
+ path: /var/lib/kubelet/plugins/nvmeof.csi.ceph.com
+ type: DirectoryOrCreate
+ name: plugin-dir
+ - hostPath:
+ path: /var/lib/kubelet/plugins
+ type: Directory
+ name: plugin-mount-dir
+ - hostPath:
+ path: /var/lib/kubelet/pods
+ type: Directory
+ name: pods-mount-dir
+ - hostPath:
+ path: /var/lib/kubelet/plugins_registry
+ type: Directory
+ name: registration-dir
+ - hostPath:
+ path: /etc/selinux
+ type: ""
+ name: etc-selinux
+ - name: oidc-token
+ projected:
+ defaultMode: 420
+ sources:
+ - serviceAccountToken:
+ audience: ceph-csi-kms
+ expirationSeconds: 3600
+ path: oidc-token
+ - hostPath:
+ path: /var/lib/cephcsi/nvmeof.csi.ceph.com-nodeplugin
+ type: DirectoryOrCreate
+ name: logs-dir
+ # Log rotation ConfigMap - optional, can be created if log rotation is needed
+ # - configMap:
+ # defaultMode: 420
+ # name: nvmeof.csi.ceph.com-logrotate-config
+ # name: log-rotate-dir
+ updateStrategy:
+ rollingUpdate:
+ maxSurge: 0
+ maxUnavailable: 1
+ type: RollingUpdate
diff --git a/deploy/examples/csi/nvmeof/nvmeof-pool.yaml b/deploy/examples/csi/nvmeof/nvmeof-pool.yaml
new file mode 100644
index 000000000..3db57b6e8
--- /dev/null
+++ b/deploy/examples/csi/nvmeof/nvmeof-pool.yaml
@@ -0,0 +1,9 @@
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+ name: nvmeof
+ namespace: rook-ceph # namespace:cluster
+spec:
+ failureDomain: host
+ replicated:
+ size: 3
diff --git a/deploy/examples/csi/nvmeof/pod.yaml b/deploy/examples/csi/nvmeof/pod.yaml
new file mode 100644
index 000000000..396c2a9f4
--- /dev/null
+++ b/deploy/examples/csi/nvmeof/pod.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nvmeof-test-pod
+ namespace: default
+spec:
+ containers:
+ - name: test-container
+ image: busybox
+ command: ["/bin/sh"]
+ args:
+ [
+ "-c",
+ "while true; do echo 'Testing NVMe-oF mount'; df -h /mnt/nvmeof; sleep 45; done",
+ ]
+ volumeMounts:
+ - name: nvmeof-volume
+ mountPath: /mnt/nvmeof
+ volumes:
+ - name: nvmeof-volume
+ persistentVolumeClaim:
+ claimName: nvmeof-external-volume
+ restartPolicy: Never
diff --git a/deploy/examples/csi/nvmeof/provisioner.yaml b/deploy/examples/csi/nvmeof/provisioner.yaml
new file mode 100644
index 000000000..225e1cfbb
--- /dev/null
+++ b/deploy/examples/csi/nvmeof/provisioner.yaml
@@ -0,0 +1,336 @@
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+ name: nvmeof-csi-provisioner
+ # replace with non-default namespace name
+ namespace: rook-ceph
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: nvmeof-external-provisioner-runner
+rules:
+ - apiGroups: [""]
+ resources: ["nodes"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["list", "watch", "create", "update", "patch"]
+ - apiGroups: [""]
+ resources: ["persistentvolumes"]
+ verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
+ - apiGroups: [""]
+ resources: ["persistentvolumeclaims"]
+ verbs: ["get", "list", "watch", "update"]
+ - apiGroups: [""]
+ resources: ["persistentvolumeclaims/status"]
+ verbs: ["update", "patch"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["storageclasses"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshots"]
+ verbs: ["get", "list", "watch", "update", "patch", "create"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshots/status"]
+ verbs: ["get", "list", "patch"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotcontents"]
+ verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotclasses"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["volumeattachments"]
+ verbs: ["get", "list", "watch", "update", "patch"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["volumeattachments/status"]
+ verbs: ["patch"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["csinodes"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotcontents/status"]
+ verbs: ["update", "patch"]
+ - apiGroups: [""]
+ resources: ["configmaps"]
+ verbs: ["get"]
+ - apiGroups: [""]
+ resources: ["serviceaccounts"]
+ verbs: ["get"]
+ - apiGroups: [""]
+ resources: ["serviceaccounts/token"]
+ verbs: ["create"]
+ - apiGroups: ["groupsnapshot.storage.k8s.io"]
+ resources: ["volumegroupsnapshotclasses"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["groupsnapshot.storage.k8s.io"]
+ resources: ["volumegroupsnapshotcontents"]
+ verbs: ["get", "list", "watch", "update", "patch"]
+ - apiGroups: ["groupsnapshot.storage.k8s.io"]
+ resources: ["volumegroupsnapshotcontents/status"]
+ verbs: ["update", "patch"]
+ - apiGroups: ["replication.storage.openshift.io"]
+ resources: ["volumegroupreplicationcontents"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["replication.storage.openshift.io"]
+ resources: ["volumegroupreplicationclasses"]
+ verbs: ["get"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["volumeattributesclasses"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["persistentvolumeclaims/status"]
+ verbs: ["patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: nvmeof-csi-provisioner-role
+subjects:
+ - kind: ServiceAccount
+ name: nvmeof-csi-provisioner
+ # replace with non-default namespace name
+ namespace: rook-ceph
+roleRef:
+ kind: ClusterRole
+ name: nvmeof-external-provisioner-runner
+ apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ # replace with non-default namespace name
+ namespace: rook-ceph
+ name: nvmeof-external-provisioner-cfg
+rules:
+ - apiGroups: [""]
+ resources: ["configmaps"]
+ verbs: ["get", "list", "watch", "create", "update", "delete"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: nvmeof-csi-provisioner-role-cfg
+ # replace with non-default namespace name
+ namespace: rook-ceph
+subjects:
+ - kind: ServiceAccount
+ name: nvmeof-csi-provisioner
+ # replace with non-default namespace name
+ namespace: rook-ceph
+roleRef:
+ kind: Role
+ name: nvmeof-external-provisioner-cfg
+ apiGroup: rbac.authorization.k8s.io
+# ---
+# # ssc based on the Ceph-CSI ssc
+# # the "users:" contains a reference to the "rook-ceph" namespace
+# #
+# kind: SecurityContextConstraints
+# apiVersion: security.openshift.io/v1
+# metadata:
+# name: "ceph-nvmeof-provisioner"
+# # To allow running privilegedContainers
+# allowPrivilegedContainer: true
+# # SYS_ADMIN is needed for rbd to execute rbd map command
+# allowedCapabilities:
+# - "SYS_ADMIN"
+# # Set to false as we write to RootFilesystem inside container
+# readOnlyRootFilesystem: false
+# requiredDropCapabilities:
+# - ALL
+# # The type of volumes which are mounted to csi pods
+# volumes:
+# - configMap
+# - projected
+# - emptyDir
+# runAsUser:
+# type: RunAsAny
+# seLinuxContext:
+# type: RunAsAny
+# users:
+# # A user needs to be added for each service account.
+# - "system:serviceaccount:rook-ceph:nvmeof-csi-provisioner"
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+ name: csi-nvmeofplugin-provisioner
+ # replace with non-default namespace name
+ namespace: rook-ceph
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: csi-nvmeofplugin-provisioner
+ template:
+ metadata:
+ annotations:
+ openshift.io/scc: rook-ceph-csi
+ labels:
+ app: csi-nvmeofplugin-provisioner
+ spec:
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values:
+ - csi-nvmeofplugin-provisioner
+ topologyKey: "kubernetes.io/hostname"
+ serviceAccountName: nvmeof-csi-provisioner
+ priorityClassName: system-cluster-critical
+ containers:
+ - name: csi-nvmeofplugin
+ # for stable functionality replace canary with latest release version
+ # image: quay.io/gdidi/cephcsi:nvmeof
+ image: quay.io/cephcsi/cephcsi:v3.16.0
+ command: ["/usr/local/bin/cephcsi"]
+ args:
+ - "--nodeid=$(NODE_ID)"
+ - "--type=nvmeof"
+ - "--controllerserver=true"
+ - "--endpoint=$(CSI_ENDPOINT)"
+ - "--v=5"
+ - "--drivername=nvmeof.csi.ceph.com"
+ - "--pidlimit=-1"
+ - "--rbdhardmaxclonedepth=8"
+ - "--rbdsoftmaxclonedepth=4"
+ - "--enableprofiling=false"
+ - "--setmetadata=true"
+ env:
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: NODE_ID
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: CSI_ENDPOINT
+ value: unix:///csi/csi-provisioner.sock
+ imagePullPolicy: "IfNotPresent"
+ securityContext:
+ privileged: true
+ capabilities:
+ drop:
+ - ALL
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ - name: ceph-csi-config
+ mountPath: /etc/ceph-csi-config/
+ - name: keys-tmp-dir
+ mountPath: /tmp/csi/keys
+ - name: oidc-token
+ mountPath: /run/secrets/tokens
+ readOnly: true
+ - name: csi-provisioner
+ # image: registry.k8s.io/sig-storage/csi-provisioner:v5.1.0
+ image: quay.io/nixpanic/csi-provisioner:pr1440
+ args:
+ - "--csi-address=$(ADDRESS)"
+ - "--v=1"
+ - "--timeout=150s"
+ - "--retry-interval-start=500ms"
+ - "--leader-election=true"
+ - "--feature-gates=HonorPVReclaimPolicy=true"
+ - "--prevent-volume-mode-conversion=true"
+ # if fstype is not specified in storageclass, ext4 is default
+ - "--default-fstype=ext4"
+ - "--extra-create-metadata=true"
+ - "--immediate-topology=false"
+ - "--http-endpoint=$(POD_IP):8090"
+ env:
+ - name: ADDRESS
+ value: unix:///csi/csi-provisioner.sock
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ imagePullPolicy: "IfNotPresent"
+ ports:
+ - containerPort: 8090
+ name: provisioner
+ protocol: TCP
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ - name: csi-attacher
+ image: registry.k8s.io/sig-storage/csi-attacher:v4.8.0
+ args:
+ - "--v=1"
+ - "--csi-address=$(ADDRESS)"
+ - "--leader-election=true"
+ - "--retry-interval-start=500ms"
+ - "--default-fstype=ext4"
+ - "--http-endpoint=$(POD_IP):8093"
+ env:
+ - name: ADDRESS
+ value: /csi/csi-provisioner.sock
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ imagePullPolicy: "IfNotPresent"
+ ports:
+ - containerPort: 8093
+ name: attacher
+ protocol: TCP
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ - name: csi-resizer
+ image: quay.io/nixpanic/csi-resizer:pr544
+ # image: registry.k8s.io/sig-storage/csi-resizer:v1.12.0
+ args:
+ - "--csi-address=$(ADDRESS)"
+ - "--v=5"
+ - "--leader-election=true"
+ - "--handle-volume-inuse-error=false"
+ - "--feature-gates=VolumeAttributesClass=true"
+ env:
+ - name: ADDRESS
+ value: unix:///csi/csi-provisioner.sock
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ volumes:
+ - name: socket-dir
+ emptyDir:
+ medium: "Memory"
+ # FIXED: Mount rook-ceph-csi-config instead of rook-ceph-nvmeof-*-config
+ # This ConfigMap contains the cluster configuration with monitor endpoints
+ # that Ceph-CSI needs to connect to the Ceph cluster
+ - name: ceph-csi-config
+ configMap:
+ name: rook-ceph-csi-config
+ items:
+ - key: csi-cluster-config-json
+ path: config.json
+ - name: keys-tmp-dir
+ emptyDir:
+ medium: "Memory"
+ # Only if nvmeof support encryption
+ - name: oidc-token
+ projected:
+ sources:
+ - serviceAccountToken:
+ path: oidc-token
+ expirationSeconds: 3600
+ audience: ceph-csi-kms
diff --git a/deploy/examples/csi/nvmeof/pvc.yaml b/deploy/examples/csi/nvmeof/pvc.yaml
new file mode 100644
index 000000000..7578de8f7
--- /dev/null
+++ b/deploy/examples/csi/nvmeof/pvc.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: nvmeof-external-volume
+ namespace: default
+spec:
+ storageClassName: ceph-nvmeof
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 128Mi
diff --git a/deploy/examples/csi/nvmeof/storageclass.yaml b/deploy/examples/csi/nvmeof/storageclass.yaml
new file mode 100644
index 000000000..a686500c0
--- /dev/null
+++ b/deploy/examples/csi/nvmeof/storageclass.yaml
@@ -0,0 +1,31 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: ceph-nvmeof
+parameters:
+ clusterID: rook-ceph
+ pool: nvmeof
+ subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph
+ # Management API - talks to gateway to create subsystems/namespaces
+ nvmeofGatewayAddress: "172.30.47.120"
+ nvmeofGatewayPort: "5500"
+ # Data Plane - worker nodes connect here for actual I/O
+ # List ALL gateway pods for HA and multipath
+ listeners: |
+ [
+ {
+ "address": "10.129.2.36",
+ "port": 4420,
+ "hostname": "rook-ceph-nvmeof-nvmeof-a"
+ }
+ ]
+ csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
+ csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+ csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
+ csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+ imageFormat: "2"
+ imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff
+provisioner: nvmeof.csi.ceph.com
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: false
diff --git a/deploy/examples/nvmeof-test.yaml b/deploy/examples/nvmeof-test.yaml
new file mode 100644
index 000000000..58a4e1c95
--- /dev/null
+++ b/deploy/examples/nvmeof-test.yaml
@@ -0,0 +1,13 @@
+# This example is for Ceph v20 and above only
+
+apiVersion: ceph.rook.io/v1
+kind: CephNVMeOFGateway
+metadata:
+ name: nvmeof
+ namespace: rook-ceph # namespace:cluster
+spec:
+ image: quay.io/ceph/nvmeof:1.5
+ pool: nvmeof
+ group: group-a
+ instances: 1
+ hostNetwork: false
diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml
index 373e076d1..df4be3118 100644
--- a/deploy/examples/operator-openshift.yaml
+++ b/deploy/examples/operator-openshift.yaml
@@ -18,7 +18,7 @@ allowHostNetwork: false
# set to true if running rook with the provider as host
allowHostPorts: false
priority:
-allowedCapabilities: ["MKNOD"]
+allowedCapabilities: ["MKNOD", "SYS_ADMIN", "NET_RAW"]
allowHostIPC: true
readOnlyRootFilesystem: false
# drop all default privileges
@@ -49,6 +49,7 @@ users:
- system:serviceaccount:rook-ceph:rook-ceph-mgr # serviceaccount:namespace:cluster
- system:serviceaccount:rook-ceph:rook-ceph-osd # serviceaccount:namespace:cluster
- system:serviceaccount:rook-ceph:rook-ceph-rgw # serviceaccount:namespace:cluster
+ - system:serviceaccount:rook-ceph:rook-ceph-nvmeof # serviceaccount:namespace:cluster
---
# scc for the CSI driver
kind: SecurityContextConstraints
@@ -97,6 +98,7 @@ users:
- system:serviceaccount:rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator
- system:serviceaccount:rook-ceph:rook-csi-nfs-plugin-sa # serviceaccount:namespace:operator
- system:serviceaccount:rook-ceph:rook-csi-nfs-provisioner-sa # serviceaccount:namespace:operator
+ - system:serviceaccount:rook-ceph:nvmeof-csi-provisioner # serviceaccount:namespace:operator
# Service accounts for the csi operator
- system:serviceaccount:rook-ceph:ceph-csi-cephfs-ctrlplugin-sa # serviceaccount:namespace:operator
- system:serviceaccount:rook-ceph:ceph-csi-cephfs-nodeplugin-sa # serviceaccount:namespace:operator
diff --git a/pkg/apis/ceph.rook.io/v1/nvmeof.go b/pkg/apis/ceph.rook.io/v1/nvmeof.go
new file mode 100644
index 000000000..1d2b28118
--- /dev/null
+++ b/pkg/apis/ceph.rook.io/v1/nvmeof.go
@@ -0,0 +1,26 @@
+/*
+Copyright 2024 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+// IsHostNetwork returns true if host networking is enabled for this gateway.
+// If not explicitly set on the gateway spec, it inherits from the cluster spec.
+func (n *CephNVMeOFGateway) IsHostNetwork(c *ClusterSpec) bool {
+ if n.Spec.HostNetwork != nil {
+ return *n.Spec.HostNetwork
+ }
+ return c.Network.IsHost()
+}
diff --git a/pkg/apis/ceph.rook.io/v1/register.go b/pkg/apis/ceph.rook.io/v1/register.go
index 1eebf7949..488e85ffd 100644
--- a/pkg/apis/ceph.rook.io/v1/register.go
+++ b/pkg/apis/ceph.rook.io/v1/register.go
@@ -65,6 +65,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&CephFilesystemList{},
&CephNFS{},
&CephNFSList{},
+ &CephNVMeOFGateway{},
+ &CephNVMeOFGatewayList{},
&CephObjectStore{},
&CephObjectStoreList{},
&CephObjectStoreUser{},
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 3732c2fd0..aa5b267af 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -3929,3 +3929,125 @@ const (
// Always means the Ceph COSI driver will be deployed even if the object store is not present
COSIDeploymentStrategyAlways COSIDeploymentStrategy = "Always"
)
+
+// +genclient
+// +genclient:noStatus
+// +kubebuilder:resource:shortName=nvmeof,path=cephnvmeofgateways
+// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+// +kubebuilder:subresource:status
+//
+// CephNVMeOFGateway represents a Ceph NVMe-oF Gateway
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+type CephNVMeOFGateway struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ObjectMeta `json:"metadata"`
+ Spec NVMeOFGatewaySpec `json:"spec"`
+ // +optional
+ Status *NVMeOFGatewayStatus `json:"status,omitempty"`
+}
+
+// NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway
+type NVMeOFGatewayStatus struct {
+ Status `json:",inline"`
+ Cephx LocalCephxStatus `json:"cephx,omitempty"`
+}
+
+// CephNVMeOFGatewayList represents a list of Ceph NVMe-oF Gateways
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+type CephNVMeOFGatewayList struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ListMeta `json:"metadata"`
+ Items []CephNVMeOFGateway `json:"items"`
+}
+
+// NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway
+type NVMeOFGatewaySpec struct {
+ // Image is the container image to use for the NVMe-oF gateway daemon.
+ // For example, quay.io/ceph/nvmeof:1.5
+ // +kubebuilder:validation:MinLength=1
+ Image string `json:"image"`
+
+ // The number of active gateway instances
+ // +kubebuilder:validation:Minimum=1
+ Instances int `json:"instances"`
+
+ // Pool is the RADOS pool where NVMe-oF configuration is stored
+ // +kubebuilder:validation:MinLength=1
+ Pool string `json:"pool"`
+
+ // Group is the gateway group name for high availability (ANA group)
+ // +kubebuilder:validation:MinLength=1
+ Group string `json:"group"`
+
+ // ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration
+ // If not specified, a default configuration will be generated
+ // +optional
+ // +kubebuilder:validation:MinLength=1
+ ConfigMapRef string `json:"configMapRef,omitempty"`
+
+ // NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration
+ // This allows users to override or add configuration options without needing to manage a ConfigMap
+ // +optional
+ NVMeOFConfig map[string]map[string]string `json:"nvmeofConfig,omitempty"`
+
+ // The affinity to place the gateway pods
+ // +optional
+ Placement Placement `json:"placement,omitempty"`
+
+ // The annotations-related configuration to add/set on each Pod related object.
+ // +optional
+ Annotations Annotations `json:"annotations,omitempty"`
+
+ // The labels-related configuration to add/set on each Pod related object.
+ // +optional
+ Labels Labels `json:"labels,omitempty"`
+
+ // Resources set resource requests and limits
+ // +optional
+ Resources v1.ResourceRequirements `json:"resources,omitempty"`
+
+ // PriorityClassName sets the priority class on the pods
+ // +optional
+ PriorityClassName string `json:"priorityClassName,omitempty"`
+
+ // Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.
+ // +optional
+ HostNetwork *bool `json:"hostNetwork,omitempty"`
+
+ // Ports configuration for the NVMe-oF gateway
+ // +optional
+ Ports *NVMeOFGatewayPorts `json:"ports,omitempty"`
+
+ // A liveness-probe to verify that gateway has valid run-time state.
+ // If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
+ // +optional
+ LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"`
+}
+
+// NVMeOFGatewayPorts represents the port configuration for NVMe-oF gateway
+type NVMeOFGatewayPorts struct {
+ // IOPort is the port for NVMe-oF IO traffic (default: 4420)
+ // +kubebuilder:validation:Minimum=1
+ // +kubebuilder:validation:Maximum=65535
+ // +optional
+ IOPort int32 `json:"ioPort,omitempty"`
+
+ // GatewayPort is the port for the gateway service (default: 5500)
+ // +kubebuilder:validation:Minimum=1
+ // +kubebuilder:validation:Maximum=65535
+ // +optional
+ GatewayPort int32 `json:"gatewayPort,omitempty"`
+
+ // MonitorPort is the port for the monitor service (default: 5499)
+ // +kubebuilder:validation:Minimum=1
+ // +kubebuilder:validation:Maximum=65535
+ // +optional
+ MonitorPort int32 `json:"monitorPort,omitempty"`
+
+ // DiscoveryPort is the port for discovery service (default: 8009)
+ // +kubebuilder:validation:Minimum=1
+ // +kubebuilder:validation:Maximum=65535
+ // +optional
+ DiscoveryPort int32 `json:"discoveryPort,omitempty"`
+}
diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
index dc4fc5f2b..c480a40ab 100644
--- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
@@ -1459,6 +1459,71 @@ func (in *CephNFSList) DeepCopyObject() runtime.Object {
return nil
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CephNVMeOFGateway) DeepCopyInto(out *CephNVMeOFGateway) {
+ *out = *in
+ out.TypeMeta = in.TypeMeta
+ in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+ in.Spec.DeepCopyInto(&out.Spec)
+ if in.Status != nil {
+ in, out := &in.Status, &out.Status
+ *out = new(NVMeOFGatewayStatus)
+ (*in).DeepCopyInto(*out)
+ }
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephNVMeOFGateway.
+func (in *CephNVMeOFGateway) DeepCopy() *CephNVMeOFGateway {
+ if in == nil {
+ return nil
+ }
+ out := new(CephNVMeOFGateway)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CephNVMeOFGateway) DeepCopyObject() runtime.Object {
+ if c := in.DeepCopy(); c != nil {
+ return c
+ }
+ return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CephNVMeOFGatewayList) DeepCopyInto(out *CephNVMeOFGatewayList) {
+ *out = *in
+ out.TypeMeta = in.TypeMeta
+ in.ListMeta.DeepCopyInto(&out.ListMeta)
+ if in.Items != nil {
+ in, out := &in.Items, &out.Items
+ *out = make([]CephNVMeOFGateway, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephNVMeOFGatewayList.
+func (in *CephNVMeOFGatewayList) DeepCopy() *CephNVMeOFGatewayList {
+ if in == nil {
+ return nil
+ }
+ out := new(CephNVMeOFGatewayList)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CephNVMeOFGatewayList) DeepCopyObject() runtime.Object {
+ if c := in.DeepCopy(); c != nil {
+ return c
+ }
+ return nil
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CephObjectRealm) DeepCopyInto(out *CephObjectRealm) {
*out = *in
@@ -3763,6 +3828,104 @@ func (in *NFSStatus) DeepCopy() *NFSStatus {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NVMeOFGatewayPorts) DeepCopyInto(out *NVMeOFGatewayPorts) {
+ *out = *in
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NVMeOFGatewayPorts.
+func (in *NVMeOFGatewayPorts) DeepCopy() *NVMeOFGatewayPorts {
+ if in == nil {
+ return nil
+ }
+ out := new(NVMeOFGatewayPorts)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NVMeOFGatewaySpec) DeepCopyInto(out *NVMeOFGatewaySpec) {
+ *out = *in
+ if in.NVMeOFConfig != nil {
+ in, out := &in.NVMeOFConfig, &out.NVMeOFConfig
+ *out = make(map[string]map[string]string, len(*in))
+ for key, val := range *in {
+ var outVal map[string]string
+ if val == nil {
+ (*out)[key] = nil
+ } else {
+ in, out := &val, &outVal
+ *out = make(map[string]string, len(*in))
+ for key, val := range *in {
+ (*out)[key] = val
+ }
+ }
+ (*out)[key] = outVal
+ }
+ }
+ in.Placement.DeepCopyInto(&out.Placement)
+ if in.Annotations != nil {
+ in, out := &in.Annotations, &out.Annotations
+ *out = make(Annotations, len(*in))
+ for key, val := range *in {
+ (*out)[key] = val
+ }
+ }
+ if in.Labels != nil {
+ in, out := &in.Labels, &out.Labels
+ *out = make(Labels, len(*in))
+ for key, val := range *in {
+ (*out)[key] = val
+ }
+ }
+ in.Resources.DeepCopyInto(&out.Resources)
+ if in.HostNetwork != nil {
+ in, out := &in.HostNetwork, &out.HostNetwork
+ *out = new(bool)
+ **out = **in
+ }
+ if in.Ports != nil {
+ in, out := &in.Ports, &out.Ports
+ *out = new(NVMeOFGatewayPorts)
+ **out = **in
+ }
+ if in.LivenessProbe != nil {
+ in, out := &in.LivenessProbe, &out.LivenessProbe
+ *out = new(ProbeSpec)
+ (*in).DeepCopyInto(*out)
+ }
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NVMeOFGatewaySpec.
+func (in *NVMeOFGatewaySpec) DeepCopy() *NVMeOFGatewaySpec {
+ if in == nil {
+ return nil
+ }
+ out := new(NVMeOFGatewaySpec)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NVMeOFGatewayStatus) DeepCopyInto(out *NVMeOFGatewayStatus) {
+ *out = *in
+ in.Status.DeepCopyInto(&out.Status)
+ out.Cephx = in.Cephx
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NVMeOFGatewayStatus.
+func (in *NVMeOFGatewayStatus) DeepCopy() *NVMeOFGatewayStatus {
+ if in == nil {
+ return nil
+ }
+ out := new(NVMeOFGatewayStatus)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NamedBlockPoolSpec) DeepCopyInto(out *NamedBlockPoolSpec) {
*out = *in
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go
index d500b7bc7..dc7c2faa4 100644
--- a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go
@@ -39,6 +39,7 @@ type CephV1Interface interface {
CephFilesystemMirrorsGetter
CephFilesystemSubVolumeGroupsGetter
CephNFSesGetter
+ CephNVMeOFGatewaysGetter
CephObjectRealmsGetter
CephObjectStoresGetter
CephObjectStoreUsersGetter
@@ -96,6 +97,10 @@ func (c *CephV1Client) CephNFSes(namespace string) CephNFSInterface {
return newCephNFSes(c, namespace)
}
+func (c *CephV1Client) CephNVMeOFGateways(namespace string) CephNVMeOFGatewayInterface {
+ return newCephNVMeOFGateways(c, namespace)
+}
+
func (c *CephV1Client) CephObjectRealms(namespace string) CephObjectRealmInterface {
return newCephObjectRealms(c, namespace)
}
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephnvmeofgateway.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephnvmeofgateway.go
new file mode 100644
index 000000000..9eab2e2c4
--- /dev/null
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephnvmeofgateway.go
@@ -0,0 +1,68 @@
+/*
+Copyright 2018 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+Â Â Â Â http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1
+
+import (
+ context "context"
+
+ cephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ scheme "github.com/rook/rook/pkg/client/clientset/versioned/scheme"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ types "k8s.io/apimachinery/pkg/types"
+ watch "k8s.io/apimachinery/pkg/watch"
+ gentype "k8s.io/client-go/gentype"
+)
+
+// CephNVMeOFGatewaysGetter has a method to return a CephNVMeOFGatewayInterface.
+// A group's client should implement this interface.
+type CephNVMeOFGatewaysGetter interface {
+ CephNVMeOFGateways(namespace string) CephNVMeOFGatewayInterface
+}
+
+// CephNVMeOFGatewayInterface has methods to work with CephNVMeOFGateway resources.
+type CephNVMeOFGatewayInterface interface {
+ Create(ctx context.Context, cephNVMeOFGateway *cephrookiov1.CephNVMeOFGateway, opts metav1.CreateOptions) (*cephrookiov1.CephNVMeOFGateway, error)
+ Update(ctx context.Context, cephNVMeOFGateway *cephrookiov1.CephNVMeOFGateway, opts metav1.UpdateOptions) (*cephrookiov1.CephNVMeOFGateway, error)
+ Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
+ DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
+ Get(ctx context.Context, name string, opts metav1.GetOptions) (*cephrookiov1.CephNVMeOFGateway, error)
+ List(ctx context.Context, opts metav1.ListOptions) (*cephrookiov1.CephNVMeOFGatewayList, error)
+ Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
+ Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *cephrookiov1.CephNVMeOFGateway, err error)
+ CephNVMeOFGatewayExpansion
+}
+
+// cephNVMeOFGateways implements CephNVMeOFGatewayInterface
+type cephNVMeOFGateways struct {
+ *gentype.ClientWithList[*cephrookiov1.CephNVMeOFGateway, *cephrookiov1.CephNVMeOFGatewayList]
+}
+
+// newCephNVMeOFGateways returns a CephNVMeOFGateways
+func newCephNVMeOFGateways(c *CephV1Client, namespace string) *cephNVMeOFGateways {
+ return &cephNVMeOFGateways{
+ gentype.NewClientWithList[*cephrookiov1.CephNVMeOFGateway, *cephrookiov1.CephNVMeOFGatewayList](
+ "cephnvmeofgateways",
+ c.RESTClient(),
+ scheme.ParameterCodec,
+ namespace,
+ func() *cephrookiov1.CephNVMeOFGateway { return &cephrookiov1.CephNVMeOFGateway{} },
+ func() *cephrookiov1.CephNVMeOFGatewayList { return &cephrookiov1.CephNVMeOFGatewayList{} },
+ ),
+ }
+}
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go
index 6c6985e2a..5986773f2 100644
--- a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go
@@ -72,6 +72,10 @@ func (c *FakeCephV1) CephNFSes(namespace string) v1.CephNFSInterface {
return newFakeCephNFSes(c, namespace)
}
+func (c *FakeCephV1) CephNVMeOFGateways(namespace string) v1.CephNVMeOFGatewayInterface {
+ return newFakeCephNVMeOFGateways(c, namespace)
+}
+
func (c *FakeCephV1) CephObjectRealms(namespace string) v1.CephObjectRealmInterface {
return newFakeCephObjectRealms(c, namespace)
}
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephnvmeofgateway.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephnvmeofgateway.go
new file mode 100644
index 000000000..e09ce287e
--- /dev/null
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephnvmeofgateway.go
@@ -0,0 +1,52 @@
+/*
+Copyright 2018 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+Â Â Â Â http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+ v1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ cephrookiov1 "github.com/rook/rook/pkg/client/clientset/versioned/typed/ceph.rook.io/v1"
+ gentype "k8s.io/client-go/gentype"
+)
+
+// fakeCephNVMeOFGateways implements CephNVMeOFGatewayInterface
+type fakeCephNVMeOFGateways struct {
+ *gentype.FakeClientWithList[*v1.CephNVMeOFGateway, *v1.CephNVMeOFGatewayList]
+ Fake *FakeCephV1
+}
+
+func newFakeCephNVMeOFGateways(fake *FakeCephV1, namespace string) cephrookiov1.CephNVMeOFGatewayInterface {
+ return &fakeCephNVMeOFGateways{
+ gentype.NewFakeClientWithList[*v1.CephNVMeOFGateway, *v1.CephNVMeOFGatewayList](
+ fake.Fake,
+ namespace,
+ v1.SchemeGroupVersion.WithResource("cephnvmeofgateways"),
+ v1.SchemeGroupVersion.WithKind("CephNVMeOFGateway"),
+ func() *v1.CephNVMeOFGateway { return &v1.CephNVMeOFGateway{} },
+ func() *v1.CephNVMeOFGatewayList { return &v1.CephNVMeOFGatewayList{} },
+ func(dst, src *v1.CephNVMeOFGatewayList) { dst.ListMeta = src.ListMeta },
+ func(list *v1.CephNVMeOFGatewayList) []*v1.CephNVMeOFGateway {
+ return gentype.ToPointerSlice(list.Items)
+ },
+ func(list *v1.CephNVMeOFGatewayList, items []*v1.CephNVMeOFGateway) {
+ list.Items = gentype.FromPointerSlice(items)
+ },
+ ),
+ fake,
+ }
+}
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go
index 82f716799..cbd8ec6c4 100644
--- a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go
@@ -40,6 +40,8 @@ type CephFilesystemSubVolumeGroupExpansion interface{}
type CephNFSExpansion interface{}
+type CephNVMeOFGatewayExpansion interface{}
+
type CephObjectRealmExpansion interface{}
type CephObjectStoreExpansion interface{}
diff --git a/pkg/client/informers/externalversions/ceph.rook.io/v1/cephnvmeofgateway.go b/pkg/client/informers/externalversions/ceph.rook.io/v1/cephnvmeofgateway.go
new file mode 100644
index 000000000..0c18e160b
--- /dev/null
+++ b/pkg/client/informers/externalversions/ceph.rook.io/v1/cephnvmeofgateway.go
@@ -0,0 +1,102 @@
+/*
+Copyright 2018 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+Â Â Â Â http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1
+
+import (
+ context "context"
+ time "time"
+
+ apiscephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ versioned "github.com/rook/rook/pkg/client/clientset/versioned"
+ internalinterfaces "github.com/rook/rook/pkg/client/informers/externalversions/internalinterfaces"
+ cephrookiov1 "github.com/rook/rook/pkg/client/listers/ceph.rook.io/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ runtime "k8s.io/apimachinery/pkg/runtime"
+ watch "k8s.io/apimachinery/pkg/watch"
+ cache "k8s.io/client-go/tools/cache"
+)
+
+// CephNVMeOFGatewayInformer provides access to a shared informer and lister for
+// CephNVMeOFGateways.
+type CephNVMeOFGatewayInformer interface {
+ Informer() cache.SharedIndexInformer
+ Lister() cephrookiov1.CephNVMeOFGatewayLister
+}
+
+type cephNVMeOFGatewayInformer struct {
+ factory internalinterfaces.SharedInformerFactory
+ tweakListOptions internalinterfaces.TweakListOptionsFunc
+ namespace string
+}
+
+// NewCephNVMeOFGatewayInformer constructs a new informer for CephNVMeOFGateway type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewCephNVMeOFGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+ return NewFilteredCephNVMeOFGatewayInformer(client, namespace, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredCephNVMeOFGatewayInformer constructs a new informer for CephNVMeOFGateway type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredCephNVMeOFGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+ return cache.NewSharedIndexInformer(
+ &cache.ListWatch{
+ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+ if tweakListOptions != nil {
+ tweakListOptions(&options)
+ }
+ return client.CephV1().CephNVMeOFGateways(namespace).List(context.Background(), options)
+ },
+ WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+ if tweakListOptions != nil {
+ tweakListOptions(&options)
+ }
+ return client.CephV1().CephNVMeOFGateways(namespace).Watch(context.Background(), options)
+ },
+ ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) {
+ if tweakListOptions != nil {
+ tweakListOptions(&options)
+ }
+ return client.CephV1().CephNVMeOFGateways(namespace).List(ctx, options)
+ },
+ WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) {
+ if tweakListOptions != nil {
+ tweakListOptions(&options)
+ }
+ return client.CephV1().CephNVMeOFGateways(namespace).Watch(ctx, options)
+ },
+ },
+ &apiscephrookiov1.CephNVMeOFGateway{},
+ resyncPeriod,
+ indexers,
+ )
+}
+
+func (f *cephNVMeOFGatewayInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+ return NewFilteredCephNVMeOFGatewayInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *cephNVMeOFGatewayInformer) Informer() cache.SharedIndexInformer {
+ return f.factory.InformerFor(&apiscephrookiov1.CephNVMeOFGateway{}, f.defaultInformer)
+}
+
+func (f *cephNVMeOFGatewayInformer) Lister() cephrookiov1.CephNVMeOFGatewayLister {
+ return cephrookiov1.NewCephNVMeOFGatewayLister(f.Informer().GetIndexer())
+}
diff --git a/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go b/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go
index 41fd56d9c..34666976b 100644
--- a/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go
+++ b/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go
@@ -46,6 +46,8 @@ type Interface interface {
CephFilesystemSubVolumeGroups() CephFilesystemSubVolumeGroupInformer
// CephNFSes returns a CephNFSInformer.
CephNFSes() CephNFSInformer
+ // CephNVMeOFGateways returns a CephNVMeOFGatewayInformer.
+ CephNVMeOFGateways() CephNVMeOFGatewayInformer
// CephObjectRealms returns a CephObjectRealmInformer.
CephObjectRealms() CephObjectRealmInformer
// CephObjectStores returns a CephObjectStoreInformer.
@@ -126,6 +128,11 @@ func (v *version) CephNFSes() CephNFSInformer {
return &cephNFSInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
}
+// CephNVMeOFGateways returns a CephNVMeOFGatewayInformer.
+func (v *version) CephNVMeOFGateways() CephNVMeOFGatewayInformer {
+ return &cephNVMeOFGatewayInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
+}
+
// CephObjectRealms returns a CephObjectRealmInformer.
func (v *version) CephObjectRealms() CephObjectRealmInformer {
return &cephObjectRealmInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go
index 107983af1..21ef1a32e 100644
--- a/pkg/client/informers/externalversions/generic.go
+++ b/pkg/client/informers/externalversions/generic.go
@@ -75,6 +75,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephFilesystemSubVolumeGroups().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("cephnfses"):
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephNFSes().Informer()}, nil
+ case v1.SchemeGroupVersion.WithResource("cephnvmeofgateways"):
+ return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephNVMeOFGateways().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("cephobjectrealms"):
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephObjectRealms().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("cephobjectstores"):
diff --git a/pkg/client/listers/ceph.rook.io/v1/cephnvmeofgateway.go b/pkg/client/listers/ceph.rook.io/v1/cephnvmeofgateway.go
new file mode 100644
index 000000000..e99251b97
--- /dev/null
+++ b/pkg/client/listers/ceph.rook.io/v1/cephnvmeofgateway.go
@@ -0,0 +1,70 @@
+/*
+Copyright 2018 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+Â Â Â Â http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1
+
+import (
+ cephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ labels "k8s.io/apimachinery/pkg/labels"
+ listers "k8s.io/client-go/listers"
+ cache "k8s.io/client-go/tools/cache"
+)
+
+// CephNVMeOFGatewayLister helps list CephNVMeOFGateways.
+// All objects returned here must be treated as read-only.
+type CephNVMeOFGatewayLister interface {
+ // List lists all CephNVMeOFGateways in the indexer.
+ // Objects returned here must be treated as read-only.
+ List(selector labels.Selector) (ret []*cephrookiov1.CephNVMeOFGateway, err error)
+ // CephNVMeOFGateways returns an object that can list and get CephNVMeOFGateways.
+ CephNVMeOFGateways(namespace string) CephNVMeOFGatewayNamespaceLister
+ CephNVMeOFGatewayListerExpansion
+}
+
+// cephNVMeOFGatewayLister implements the CephNVMeOFGatewayLister interface.
+type cephNVMeOFGatewayLister struct {
+ listers.ResourceIndexer[*cephrookiov1.CephNVMeOFGateway]
+}
+
+// NewCephNVMeOFGatewayLister returns a new CephNVMeOFGatewayLister.
+func NewCephNVMeOFGatewayLister(indexer cache.Indexer) CephNVMeOFGatewayLister {
+ return &cephNVMeOFGatewayLister{listers.New[*cephrookiov1.CephNVMeOFGateway](indexer, cephrookiov1.Resource("cephnvmeofgateway"))}
+}
+
+// CephNVMeOFGateways returns an object that can list and get CephNVMeOFGateways.
+func (s *cephNVMeOFGatewayLister) CephNVMeOFGateways(namespace string) CephNVMeOFGatewayNamespaceLister {
+ return cephNVMeOFGatewayNamespaceLister{listers.NewNamespaced[*cephrookiov1.CephNVMeOFGateway](s.ResourceIndexer, namespace)}
+}
+
+// CephNVMeOFGatewayNamespaceLister helps list and get CephNVMeOFGateways.
+// All objects returned here must be treated as read-only.
+type CephNVMeOFGatewayNamespaceLister interface {
+ // List lists all CephNVMeOFGateways in the indexer for a given namespace.
+ // Objects returned here must be treated as read-only.
+ List(selector labels.Selector) (ret []*cephrookiov1.CephNVMeOFGateway, err error)
+ // Get retrieves the CephNVMeOFGateway from the indexer for a given namespace and name.
+ // Objects returned here must be treated as read-only.
+ Get(name string) (*cephrookiov1.CephNVMeOFGateway, error)
+ CephNVMeOFGatewayNamespaceListerExpansion
+}
+
+// cephNVMeOFGatewayNamespaceLister implements the CephNVMeOFGatewayNamespaceLister
+// interface.
+type cephNVMeOFGatewayNamespaceLister struct {
+ listers.ResourceIndexer[*cephrookiov1.CephNVMeOFGateway]
+}
diff --git a/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go b/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go
index a4d3eabcb..47b6a3046 100644
--- a/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go
+++ b/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go
@@ -106,6 +106,14 @@ type CephNFSListerExpansion interface{}
// CephNFSNamespaceLister.
type CephNFSNamespaceListerExpansion interface{}
+// CephNVMeOFGatewayListerExpansion allows custom methods to be added to
+// CephNVMeOFGatewayLister.
+type CephNVMeOFGatewayListerExpansion interface{}
+
+// CephNVMeOFGatewayNamespaceListerExpansion allows custom methods to be added to
+// CephNVMeOFGatewayNamespaceLister.
+type CephNVMeOFGatewayNamespaceListerExpansion interface{}
+
// CephObjectRealmListerExpansion allows custom methods to be added to
// CephObjectRealmLister.
type CephObjectRealmListerExpansion interface{}
diff --git a/pkg/operator/ceph/cr_manager.go b/pkg/operator/ceph/cr_manager.go
index 60ae24ab3..42c7d5853 100644
--- a/pkg/operator/ceph/cr_manager.go
+++ b/pkg/operator/ceph/cr_manager.go
@@ -33,6 +33,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/file/mirror"
"github.com/rook/rook/pkg/operator/ceph/file/subvolumegroup"
"github.com/rook/rook/pkg/operator/ceph/nfs"
+ "github.com/rook/rook/pkg/operator/ceph/nvmeof"
"github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/pkg/operator/ceph/object/bucket"
"github.com/rook/rook/pkg/operator/ceph/object/cosi"
@@ -82,6 +83,7 @@ var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Conte
nfs.Add,
rbd.Add,
client.Add,
+ nvmeof.Add,
mirror.Add,
Add,
csi.Add,
diff --git a/pkg/operator/ceph/nvmeof/connectionconfig.sh b/pkg/operator/ceph/nvmeof/connectionconfig.sh
new file mode 100644
index 000000000..aa52958d0
--- /dev/null
+++ b/pkg/operator/ceph/nvmeof/connectionconfig.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -xEeuo pipefail
+
+cat << EOF > /etc/ceph/ceph.conf
+[global]
+mon_host = ${ROOK_CEPH_MON_HOST}
+log_to_stderr = true
+keyring = /etc/ceph/keyring
+EOF
+
+cp /tmp/ceph/keyring /etc/ceph
+
+chmod 444 /etc/ceph/ceph.conf
+chmod 440 /etc/ceph/keyring
+
+# ARGS: all args are assumed to be 'ceph' CLI args
+# Use "$@" to pass through Ceph CLI arguments (e.g., --connect-timeout, --mon-host, --keyring, etc.)
+# This allows easy passthru of arguments set by Rook's methods for Ceph CLI options
+
+sed -e "s/@@POD_NAME@@/${GATEWAY_NAME}/g" \
+ -e "s/@@ANA_GROUP@@/${ANA_GROUP}/g" \
+ -e "s/@@POD_IP@@/${POD_IP}/g" \
+ < /config/nvmeof.conf > /etc/ceph/nvmeof.conf
+
+ceph "$@" nvme-gw create "${GATEWAY_NAME}" "${POOL_NAME}" "${ANA_GROUP}" || true
+ceph "$@" nvme-gw show ${POOL_NAME} ${ANA_GROUP} || true
diff --git a/pkg/operator/ceph/nvmeof/controller.go b/pkg/operator/ceph/nvmeof/controller.go
new file mode 100644
index 000000000..caa6eae2a
--- /dev/null
+++ b/pkg/operator/ceph/nvmeof/controller.go
@@ -0,0 +1,587 @@
+/*
+Copyright 2025 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nvmeof
+
+import (
+ "context"
+ "fmt"
+ "reflect"
+ "strings"
+
+ csiopv1 "github.com/ceph/ceph-csi-operator/api/v1"
+ "github.com/coreos/pkg/capnslog"
+ addonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/csiaddons/v1alpha1"
+ "github.com/pkg/errors"
+ cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ "github.com/rook/rook/pkg/clusterd"
+ cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+ "github.com/rook/rook/pkg/operator/ceph/config"
+ "github.com/rook/rook/pkg/operator/ceph/config/keyring"
+ opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
+ "github.com/rook/rook/pkg/operator/ceph/reporting"
+ "github.com/rook/rook/pkg/operator/k8sutil"
+ "gopkg.in/ini.v1"
+ appsv1 "k8s.io/api/apps/v1"
+ v1 "k8s.io/api/core/v1"
+ kerrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/client-go/tools/record"
+ "k8s.io/client-go/util/retry"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/controller"
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+ "sigs.k8s.io/controller-runtime/pkg/handler"
+ "sigs.k8s.io/controller-runtime/pkg/manager"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
+ "sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+const (
+ packageName = "ceph-nvmeof-gateway"
+ controllerName = packageName + "-controller"
+)
+
+var logger = capnslog.NewPackageLogger("github.com/rook/rook", packageName)
+
+var objectsToWatch = []client.Object{
+ &v1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: v1.SchemeGroupVersion.String()}},
+ &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.String()}},
+}
+
+var controllerTypeMeta = metav1.TypeMeta{
+ Kind: reflect.TypeFor[cephv1.CephNVMeOFGateway]().Name(),
+ APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version),
+}
+
+var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion
+
+type ReconcileCephNVMeOFGateway struct {
+ client client.Client
+ scheme *runtime.Scheme
+ context *clusterd.Context
+ cephClusterSpec *cephv1.ClusterSpec
+ clusterInfo *cephclient.ClusterInfo
+ opManagerContext context.Context
+ opConfig opcontroller.OperatorConfig
+ recorder record.EventRecorder
+ shouldRotateCephxKeys bool
+}
+
+// Add creates a new CephNVMeOFGateway Controller and adds it to the Manager.
+func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
+ return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig))
+}
+
+func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
+ return &ReconcileCephNVMeOFGateway{
+ client: mgr.GetClient(),
+ scheme: mgr.GetScheme(),
+ context: context,
+ opManagerContext: opManagerContext,
+ opConfig: opConfig,
+ recorder: mgr.GetEventRecorderFor("rook-" + controllerName),
+ }
+}
+
+func watchOwnedCoreObject[T client.Object](c controller.Controller, mgr manager.Manager, obj T) error {
+ return c.Watch(
+ source.Kind(
+ mgr.GetCache(),
+ obj,
+ handler.TypedEnqueueRequestForOwner[T](
+ mgr.GetScheme(),
+ mgr.GetRESTMapper(),
+ &cephv1.CephNVMeOFGateway{},
+ ),
+ opcontroller.WatchPredicateForNonCRDObject[T](&cephv1.CephNVMeOFGateway{TypeMeta: controllerTypeMeta}, mgr.GetScheme()),
+ ),
+ )
+}
+
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+ c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
+ if err != nil {
+ return errors.Wrap(err, "failed to create controller")
+ }
+ logger.Info("successfully started")
+
+ err = addonsv1alpha1.AddToScheme(mgr.GetScheme())
+ if err != nil {
+ return errors.Wrap(err, "failed to add addonsv1alpha1 to scheme")
+ }
+
+ err = csiopv1.AddToScheme(mgr.GetScheme())
+ if err != nil {
+ return errors.Wrap(err, "failed to add csiopv1 to scheme")
+ }
+
+ err = c.Watch(
+ source.Kind(
+ mgr.GetCache(),
+ &cephv1.CephNVMeOFGateway{TypeMeta: controllerTypeMeta},
+ &handler.TypedEnqueueRequestForObject[*cephv1.CephNVMeOFGateway]{},
+ opcontroller.WatchControllerPredicate[*cephv1.CephNVMeOFGateway](mgr.GetScheme()),
+ ),
+ )
+ if err != nil {
+ return errors.Wrap(err, "failed to watch CephNVMeOFGateway CRD")
+ }
+
+ for _, t := range objectsToWatch {
+ err = watchOwnedCoreObject(c, mgr, t)
+ if err != nil {
+ return errors.Wrapf(err, "failed to watch resource type %s", t.GetObjectKind().GroupVersionKind().String())
+ }
+ }
+
+ return nil
+}
+
+// Reconcile reads the state of the cluster for a CephNVMeOFGateway object and makes changes based on the state read.
+func (r *ReconcileCephNVMeOFGateway) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
+ defer opcontroller.RecoverAndLogException()
+ reconcileResponse, cephNVMeOFGateway, err := r.reconcile(request)
+ result, reportErr := reporting.ReportReconcileResult(logger, r.recorder, request, &cephNVMeOFGateway, reconcileResponse, err)
+ return result, reportErr
+}
+
+func (r *ReconcileCephNVMeOFGateway) reconcile(request reconcile.Request) (reconcile.Result, cephv1.CephNVMeOFGateway, error) {
+ cephNVMeOFGateway := &cephv1.CephNVMeOFGateway{}
+ err := r.client.Get(r.opManagerContext, request.NamespacedName, cephNVMeOFGateway)
+ if err != nil {
+ if kerrors.IsNotFound(err) {
+ return reconcile.Result{}, *cephNVMeOFGateway, nil
+ }
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to get CephNVMeOFGateway")
+ }
+
+ observedGeneration := cephNVMeOFGateway.ObjectMeta.Generation
+
+ generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephNVMeOFGateway)
+ if err != nil {
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to add finalizer")
+ }
+ if generationUpdated {
+ return reconcile.Result{}, *cephNVMeOFGateway, nil
+ }
+
+ if cephNVMeOFGateway.Status == nil {
+ cephxUninitialized := keyring.UninitializedCephxStatus()
+ err := r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, &cephxUninitialized, k8sutil.EmptyStatus)
+ if err != nil {
+ return opcontroller.ImmediateRetryResult, *cephNVMeOFGateway, errors.Wrapf(err, "failed set empty status")
+ }
+ cephNVMeOFGateway.Status = &cephv1.NVMeOFGatewayStatus{
+ Status: cephv1.Status{},
+ Cephx: cephv1.LocalCephxStatus{Daemon: cephxUninitialized},
+ }
+ }
+
+ cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
+
+ if !isReadyToReconcile {
+ if !cephNVMeOFGateway.GetDeletionTimestamp().IsZero() && !cephClusterExists {
+ err := opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephNVMeOFGateway)
+ if err != nil {
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to remove finalizer")
+ }
+ r.recorder.Event(cephNVMeOFGateway, v1.EventTypeNormal, string(cephv1.ReconcileSucceeded), "successfully removed finalizer")
+ return reconcile.Result{}, *cephNVMeOFGateway, nil
+ }
+ return reconcileResponse, *cephNVMeOFGateway, nil
+ }
+ r.cephClusterSpec = &cephCluster.Spec
+
+ r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.cephClusterSpec)
+ if err != nil {
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to populate cluster info")
+ }
+
+ if !cephNVMeOFGateway.GetDeletionTimestamp().IsZero() {
+ logger.Infof("deleting ceph nvmeof gateway %q", cephNVMeOFGateway.Name)
+ r.recorder.Eventf(cephNVMeOFGateway, v1.EventTypeNormal, string(cephv1.ReconcileStarted), "deleting CephNVMeOFGateway %q", cephNVMeOFGateway.Name)
+
+ runningCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, config.MonType)
+ if err != nil {
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrapf(err, "failed to retrieve current ceph version")
+ }
+ r.clusterInfo.CephVersion = runningCephVersion
+
+ err = opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephNVMeOFGateway)
+ if err != nil {
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to remove finalizer")
+ }
+ r.recorder.Event(cephNVMeOFGateway, v1.EventTypeNormal, string(cephv1.ReconcileSucceeded), "successfully removed finalizer")
+ return reconcile.Result{}, *cephNVMeOFGateway, nil
+ }
+
+ runningCephVersion, desiredCephVersion, err := currentAndDesiredCephVersion(
+ r.opManagerContext, r.opConfig.Image, cephNVMeOFGateway.Namespace, controllerName,
+ k8sutil.NewOwnerInfo(cephNVMeOFGateway, r.scheme), r.context, r.cephClusterSpec, r.clusterInfo,
+ )
+ if err != nil {
+ if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
+ logger.Info(opcontroller.OperatorNotInitializedMessage)
+ return opcontroller.WaitForRequeueIfOperatorNotInitialized, *cephNVMeOFGateway, nil
+ }
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to detect ceph version")
+ }
+
+ if !cephCluster.Spec.External.Enable && !reflect.DeepEqual(*runningCephVersion, *desiredCephVersion) {
+ return opcontroller.WaitForRequeueIfCephClusterIsUpgrading, *cephNVMeOFGateway,
+ opcontroller.ErrorCephUpgradingRequeue(desiredCephVersion, runningCephVersion)
+ }
+ r.clusterInfo.CephVersion = *runningCephVersion
+
+ if err := validateGateway(cephNVMeOFGateway); err != nil {
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrapf(err, "invalid configuration")
+ }
+
+ r.shouldRotateCephxKeys, err = keyring.ShouldRotateCephxKeys(cephCluster.Spec.Security.CephX.Daemon, *runningCephVersion,
+ *desiredCephVersion, cephNVMeOFGateway.Status.Cephx.Daemon)
+ if err != nil {
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to determine if cephx keys should be rotated")
+ }
+ if r.shouldRotateCephxKeys {
+ logger.Infof("cephx keys will be rotated for %q", request.NamespacedName)
+ }
+
+ _, err = r.reconcileCreateCephNVMeOFGateway(cephNVMeOFGateway)
+ if err != nil {
+ return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to create deployments")
+ }
+
+ cephxStatus := keyring.UpdatedCephxStatus(r.shouldRotateCephxKeys, cephCluster.Spec.Security.CephX.Daemon, r.clusterInfo.CephVersion, cephNVMeOFGateway.Status.Cephx.Daemon)
+ err = r.updateStatus(observedGeneration, request.NamespacedName, &cephxStatus, k8sutil.ReadyStatus)
+ if err != nil {
+ logger.Errorf("failed to update status: %v", err)
+ return opcontroller.ImmediateRetryResult, *cephNVMeOFGateway, errors.Wrapf(err, "failed to update status")
+ }
+
+ return reconcile.Result{}, *cephNVMeOFGateway, nil
+}
+
+func (r *ReconcileCephNVMeOFGateway) reconcileCreateCephNVMeOFGateway(cephNVMeOFGateway *cephv1.CephNVMeOFGateway) (reconcile.Result, error) {
+ if r.cephClusterSpec.External.Enable {
+ _, err := opcontroller.ValidateCephVersionsBetweenLocalAndExternalClusters(r.context, r.clusterInfo)
+ if err != nil {
+ return reconcile.Result{}, errors.Wrap(err, "refusing to run new crd")
+ }
+ }
+
+ listOps := metav1.ListOptions{
+ LabelSelector: fmt.Sprintf("%s=%s,app.kubernetes.io/part-of=%s", k8sutil.AppAttr, AppName, cephNVMeOFGateway.Name),
+ }
+ deployments, err := r.context.Clientset.AppsV1().Deployments(cephNVMeOFGateway.Namespace).List(r.opManagerContext, listOps)
+ if err != nil && !kerrors.IsNotFound(err) {
+ return reconcile.Result{}, errors.Wrapf(err, "failed to list deployments")
+ }
+
+ currentGatewayCount := 0
+ if deployments != nil {
+ currentGatewayCount = len(deployments.Items)
+ }
+
+ if currentGatewayCount > cephNVMeOFGateway.Spec.Instances {
+ logger.Infof("scaling down from %d to %d", currentGatewayCount, cephNVMeOFGateway.Spec.Instances)
+ err := r.downCephNVMeOFGateway(cephNVMeOFGateway, currentGatewayCount)
+ if err != nil {
+ return reconcile.Result{}, errors.Wrapf(err, "failed to scale down")
+ }
+ }
+
+ err = r.upCephNVMeOFGateway(cephNVMeOFGateway)
+ if err != nil {
+ return reconcile.Result{}, errors.Wrapf(err, "failed to update gateway")
+ }
+
+ return reconcile.Result{}, nil
+}
+
+func (r *ReconcileCephNVMeOFGateway) upCephNVMeOFGateway(cephNVMeOFGateway *cephv1.CephNVMeOFGateway) error {
+ for i := 0; i < cephNVMeOFGateway.Spec.Instances; i++ {
+ daemonID := k8sutil.IndexToName(i)
+
+ var configMapName, configHash string
+ var err error
+
+ if cephNVMeOFGateway.Spec.ConfigMapRef == "" {
+ configMapName, configHash, err = r.createConfigMap(cephNVMeOFGateway, daemonID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to create configmap for %q", daemonID)
+ }
+ logger.Infof("configmap %q created/updated for nvmeof gateway %q instance %q with hash %q", configMapName, cephNVMeOFGateway.Name, daemonID, configHash)
+ } else {
+ configMapName = cephNVMeOFGateway.Spec.ConfigMapRef
+ configHash = "" // Will be empty for custom configmap
+ }
+
+ deployment, err := r.makeDeployment(cephNVMeOFGateway, daemonID, configMapName, configHash)
+ if err != nil {
+ return errors.Wrapf(err, "failed to make deployment for %q", daemonID)
+ }
+
+ err = controllerutil.SetControllerReference(cephNVMeOFGateway, deployment, r.scheme)
+ if err != nil {
+ return errors.Wrapf(err, "failed to set owner reference for deployment %q", deployment.Name)
+ }
+
+ _, err = k8sutil.CreateOrUpdateDeployment(r.opManagerContext, r.context.Clientset, deployment)
+ if err != nil {
+ return errors.Wrapf(err, "failed to create/update deployment for %q", daemonID)
+ }
+
+ err = r.createCephNVMeOFService(cephNVMeOFGateway, daemonID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to create service for %q", daemonID)
+ }
+ }
+
+ return nil
+}
+
+func (r *ReconcileCephNVMeOFGateway) downCephNVMeOFGateway(cephNVMeOFGateway *cephv1.CephNVMeOFGateway, currentCount int) error {
+ for i := cephNVMeOFGateway.Spec.Instances; i < currentCount; i++ {
+ daemonID := k8sutil.IndexToName(i)
+ name := instanceName(cephNVMeOFGateway, daemonID)
+
+ err := r.context.Clientset.AppsV1().Deployments(cephNVMeOFGateway.Namespace).Delete(r.opManagerContext, name, metav1.DeleteOptions{})
+ if err != nil && !kerrors.IsNotFound(err) {
+ return errors.Wrapf(err, "failed to delete deployment %q", name)
+ }
+
+ err = r.context.Clientset.CoreV1().Services(cephNVMeOFGateway.Namespace).Delete(r.opManagerContext, name, metav1.DeleteOptions{})
+ if err != nil && !kerrors.IsNotFound(err) {
+ return errors.Wrapf(err, "failed to delete service %q", name)
+ }
+ }
+
+ return nil
+}
+
+// getNVMeOFGatewayConfig generates a complete nvmeof.conf configuration file
+// with all values filled in (no placeholders). User overrides from nvmeofConfig
+// are merged on top of the default configuration.
+func getNVMeOFGatewayConfig(poolName, podName, podIP, anaGroup string, userConfig map[string]map[string]string) (string, error) {
+ cfg := ini.Empty()
+ // Set default [gateway] section
+ gatewaySection, err := cfg.NewSection("gateway")
+ if err != nil {
+ return "", errors.Wrap(err, "failed to create gateway section")
+ }
+ gatewaySection.Key("name").SetValue(podName)
+ gatewaySection.Key("group").SetValue(anaGroup)
+ gatewaySection.Key("addr").SetValue(podIP)
+ gatewaySection.Key("port").SetValue("5500")
+ gatewaySection.Key("enable_auth").SetValue("False")
+ gatewaySection.Key("state_update_notify").SetValue("True")
+ gatewaySection.Key("state_update_timeout_in_msec").SetValue("2000")
+ gatewaySection.Key("state_update_interval_sec").SetValue("5")
+ gatewaySection.Key("enable_spdk_discovery_controller").SetValue("False")
+ gatewaySection.Key("encryption_key").SetValue("/etc/ceph/encryption.key")
+ gatewaySection.Key("rebalance_period_sec").SetValue("7")
+ gatewaySection.Key("max_gws_in_grp").SetValue("16")
+ gatewaySection.Key("max_ns_to_change_lb_grp").SetValue("8")
+ gatewaySection.Key("verify_listener_ip").SetValue("False")
+ gatewaySection.Key("enable_monitor_client").SetValue("True")
+
+ // Set default [discovery] section
+ discoverySection, err := cfg.NewSection("discovery")
+ if err != nil {
+ return "", errors.Wrap(err, "failed to create discovery section")
+ }
+ discoverySection.Key("addr").SetValue("0.0.0.0")
+ discoverySection.Key("port").SetValue("8009")
+
+ // Set default [ceph] section
+ cephSection, err := cfg.NewSection("ceph")
+ if err != nil {
+ return "", errors.Wrap(err, "failed to create ceph section")
+ }
+ cephSection.Key("id").SetValue("admin")
+ cephSection.Key("pool").SetValue(poolName)
+ cephSection.Key("config_file").SetValue("/etc/ceph/ceph.conf")
+
+ // Set default [mtls] section
+ mtlsSection, err := cfg.NewSection("mtls")
+ if err != nil {
+ return "", errors.Wrap(err, "failed to create mtls section")
+ }
+ mtlsSection.Key("server_key").SetValue("./server.key")
+ mtlsSection.Key("client_key").SetValue("./client.key")
+ mtlsSection.Key("server_cert").SetValue("./server.crt")
+ mtlsSection.Key("client_cert").SetValue("./client.crt")
+
+ // Set default [spdk] section
+ spdkSection, err := cfg.NewSection("spdk")
+ if err != nil {
+ return "", errors.Wrap(err, "failed to create spdk section")
+ }
+ spdkSection.Key("bdevs_per_cluster").SetValue("32")
+ spdkSection.Key("mem_size").SetValue("4096")
+ spdkSection.Key("tgt_path").SetValue("/usr/local/bin/nvmf_tgt")
+ spdkSection.Key("timeout").SetValue("60.0")
+ spdkSection.Key("rpc_socket").SetValue("/var/tmp/spdk.sock")
+
+ // Set default [monitor] section
+ monitorSection, err := cfg.NewSection("monitor")
+ if err != nil {
+ return "", errors.Wrap(err, "failed to create monitor section")
+ }
+ monitorSection.Key("port").SetValue("5499")
+
+ // Apply user overrides
+ for sectionName, options := range userConfig {
+ section := cfg.Section(sectionName)
+ if section == nil {
+ // Section doesn't exist, create it
+ var createErr error
+ section, createErr = cfg.NewSection(sectionName)
+ if createErr != nil {
+ return "", errors.Wrapf(createErr, "failed to create section %q", sectionName)
+ }
+ }
+ for key, value := range options {
+ section.Key(key).SetValue(value)
+ }
+ }
+
+ // Write to string with proper formatting
+ var buf strings.Builder
+ _, err = cfg.WriteTo(&buf)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to write config to string")
+ }
+
+ return buf.String(), nil
+}
+
+func (r *ReconcileCephNVMeOFGateway) generateConfigMap(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) (*v1.ConfigMap, error) {
+ poolName := nvmeof.Spec.Pool
+ anaGroup := nvmeof.Spec.Group
+ podName := instanceName(nvmeof, daemonID)
+ // Use placeholder that will be replaced at runtime with actual pod IP
+ // The init container will replace @@POD_IP@@ with the actual pod IP
+ podIP := "@@POD_IP@@"
+
+ configContent, err := getNVMeOFGatewayConfig(poolName, podName, podIP, anaGroup, nvmeof.Spec.NVMeOFConfig)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to generate nvmeof config")
+ }
+ data := map[string]string{
+ "config": configContent,
+ }
+
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-%s-config", nvmeof.Name, daemonID)
+
+ configMap := &v1.ConfigMap{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: configMapName,
+ Namespace: nvmeof.Namespace,
+ Labels: getLabels(nvmeof, daemonID),
+ },
+ Data: data,
+ }
+
+ return configMap, nil
+}
+
+func (r *ReconcileCephNVMeOFGateway) createConfigMap(cephNVMeOFGateway *cephv1.CephNVMeOFGateway, daemonID string) (string, string, error) {
+ configMap, err := r.generateConfigMap(cephNVMeOFGateway, daemonID)
+ if err != nil {
+ logger.Errorf("failed to generate configmap: %v", err)
+ return "", "", err
+ }
+
+ err = controllerutil.SetControllerReference(cephNVMeOFGateway, configMap, r.scheme)
+ if err != nil {
+ logger.Errorf("failed to set owner reference: %v", err)
+ return "", "", errors.Wrapf(err, "failed to set owner reference for nvmeof configmap %q", configMap.Name)
+ }
+
+ if _, err := r.context.Clientset.CoreV1().ConfigMaps(cephNVMeOFGateway.Namespace).Create(r.opManagerContext, configMap, metav1.CreateOptions{}); err != nil {
+ if !kerrors.IsAlreadyExists(err) {
+ logger.Errorf("failed to create configmap: %v", err)
+ return "", "", errors.Wrap(err, "failed to create nvmeof config map")
+ }
+
+ if _, err = r.context.Clientset.CoreV1().ConfigMaps(cephNVMeOFGateway.Namespace).Update(r.opManagerContext, configMap, metav1.UpdateOptions{}); err != nil {
+ logger.Errorf("failed to update configmap: %v", err)
+ return "", "", errors.Wrap(err, "failed to update nvmeof config map")
+ }
+ }
+
+ configHash := k8sutil.Hash(fmt.Sprintf("%v", configMap.Data))
+ return configMap.Name, configHash, nil
+}
+
+func validateGateway(g *cephv1.CephNVMeOFGateway) error {
+ if g.Spec.Instances < 1 {
+ return errors.New("at least one gateway instance is required")
+ }
+
+ if g.Spec.Group == "" {
+ return errors.New("gateway group name is required")
+ }
+
+ if g.Spec.Pool == "" {
+ return errors.New("pool name is required")
+ }
+
+ return nil
+}
+
+func (r *ReconcileCephNVMeOFGateway) updateStatus(observedGeneration int64, namespacedName types.NamespacedName, cephxStatus *cephv1.CephxStatus, status string) error {
+ nvmeof := &cephv1.CephNVMeOFGateway{}
+
+ err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+ err := r.client.Get(r.opManagerContext, namespacedName, nvmeof)
+ if err != nil {
+ if kerrors.IsNotFound(err) {
+ return nil
+ }
+ logger.Errorf("failed to get gateway: %v", err)
+ return errors.Wrapf(err, "failed to get for status update")
+ }
+
+ if nvmeof.Status == nil {
+ nvmeof.Status = &cephv1.NVMeOFGatewayStatus{}
+ }
+
+ nvmeof.Status.Phase = status
+
+ if observedGeneration != k8sutil.ObservedGenerationNotAvailable {
+ nvmeof.Status.ObservedGeneration = observedGeneration
+ }
+
+ if cephxStatus != nil {
+ nvmeof.Status.Cephx.Daemon = *cephxStatus
+ }
+
+ if err := reporting.UpdateStatus(r.client, nvmeof); err != nil {
+ return errors.Wrapf(err, "failed to set status")
+ }
+ return nil
+ })
+ if err != nil {
+ logger.Errorf("retry loop failed: %v", err)
+ return err
+ }
+ return nil
+}
diff --git a/pkg/operator/ceph/nvmeof/controller_test.go b/pkg/operator/ceph/nvmeof/controller_test.go
new file mode 100644
index 000000000..2ce57ddfa
--- /dev/null
+++ b/pkg/operator/ceph/nvmeof/controller_test.go
@@ -0,0 +1,712 @@
+/*
+Copyright 2025 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nvmeof
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/coreos/pkg/capnslog"
+ cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
+ "github.com/rook/rook/pkg/client/clientset/versioned/scheme"
+ "github.com/rook/rook/pkg/clusterd"
+ cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+ "github.com/rook/rook/pkg/operator/ceph/version"
+ "github.com/rook/rook/pkg/operator/k8sutil"
+ "github.com/rook/rook/pkg/operator/test"
+ exectest "github.com/rook/rook/pkg/util/exec/test"
+ "github.com/stretchr/testify/assert"
+ ini "gopkg.in/ini.v1"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/client-go/tools/record"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+var (
+ name = "my-nvmeof"
+ namespace = "rook-ceph"
+)
+
+func TestCephNVMeOFGatewayController(t *testing.T) {
+ ctx := context.TODO()
+ // Set DEBUG logging
+ capnslog.SetGlobalLogLevel(capnslog.DEBUG)
+ os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
+
+ // Register operator types with the runtime scheme.
+ testScheme := scheme.Scheme
+ testScheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephNVMeOFGateway{})
+ testScheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
+
+ baseExecutor := func() *exectest.MockExecutor {
+ return &exectest.MockExecutor{
+ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
+ logger.Infof("mock execute: %s %v", command, args)
+ if command == "ceph" && args[0] == "status" {
+ return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
+ }
+ panic(fmt.Sprintf("unhandled command %s %v", command, args))
+ },
+ }
+ }
+
+ successExecutor := func(t *testing.T) *exectest.MockExecutor {
+ t.Helper()
+
+ return &exectest.MockExecutor{
+ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
+ logger.Infof("mock execute: %s %v", command, args)
+ if command == "ceph" {
+ if args[0] == "status" {
+ return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
+ }
+ if args[0] == "nvme-gw" && args[1] == "create" {
+ return "", nil
+ }
+ if args[0] == "nvme-gw" && args[1] == "show" {
+ return "", nil
+ }
+ }
+ panic(fmt.Sprintf("unhandled command %s %v", command, args))
+ },
+ }
+ }
+
+ newContext := func(executor *exectest.MockExecutor) *clusterd.Context {
+ clientset := test.New(t, 3)
+ return &clusterd.Context{
+ Executor: executor,
+ RookClientset: rookclient.NewSimpleClientset(),
+ Clientset: clientset,
+ }
+ }
+
+ baseCephNVMeOFGateway := func() *cephv1.CephNVMeOFGateway {
+ return &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: namespace,
+ Finalizers: []string{"cephnvmeofgateway.ceph.rook.io"},
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ TypeMeta: controllerTypeMeta,
+ }
+ }
+
+ cephClusterNotReady := func() *cephv1.CephCluster {
+ return &cephv1.CephCluster{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: namespace,
+ Namespace: namespace,
+ },
+ Status: cephv1.ClusterStatus{
+ Phase: "",
+ CephStatus: &cephv1.CephStatus{
+ Health: "",
+ },
+ },
+ }
+ }
+
+ cephClusterReady := func(clusterCtx *clusterd.Context) *cephv1.CephCluster {
+ cephCluster := cephClusterNotReady()
+ cephCluster.Status.Phase = k8sutil.ReadyStatus
+ cephCluster.Status.CephStatus.Health = "HEALTH_OK"
+
+ secrets := map[string][]byte{
+ "fsid": []byte(name),
+ "mon-secret": []byte("monsecret"),
+ "admin-secret": []byte("adminsecret"),
+ }
+ secret := &v1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "rook-ceph-mon",
+ Namespace: namespace,
+ },
+ Data: secrets,
+ Type: k8sutil.RookType,
+ }
+ _, err := clusterCtx.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
+ if err != nil {
+ panic(fmt.Sprintf("should be no error: %v", err))
+ }
+
+ return cephCluster
+ }
+
+ newControllerClient := func(objects ...runtime.Object) client.WithWatch {
+ return fake.NewClientBuilder().WithScheme(testScheme).WithRuntimeObjects(objects...).Build()
+ }
+
+ newReconcile := func(clusterCtx *clusterd.Context, cl client.WithWatch) *ReconcileCephNVMeOFGateway {
+ return &ReconcileCephNVMeOFGateway{client: cl, scheme: testScheme, context: clusterCtx, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)}
+ }
+
+ req := reconcile.Request{
+ NamespacedName: types.NamespacedName{
+ Name: name,
+ Namespace: namespace,
+ },
+ }
+
+ currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) {
+ return &version.Squid, &version.Squid, nil
+ }
+
+ t.Run("error - no ceph cluster", func(t *testing.T) {
+ cCtx := newContext(baseExecutor())
+ cl := newControllerClient(baseCephNVMeOFGateway())
+ r := newReconcile(cCtx, cl)
+
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Greater(t, res.RequeueAfter, time.Duration(0))
+ })
+
+ t.Run("error - ceph cluster not ready", func(t *testing.T) {
+ cCtx := newContext(baseExecutor())
+ cl := newControllerClient(baseCephNVMeOFGateway(), cephClusterNotReady())
+ r := newReconcile(cCtx, cl)
+
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Greater(t, res.RequeueAfter, time.Duration(0))
+ })
+
+ t.Run("error - invalid gateway spec", func(t *testing.T) {
+ t.Run("instances less than 1 should error", func(t *testing.T) {
+ cCtx := newContext(successExecutor(t))
+ nvmeof := baseCephNVMeOFGateway()
+ nvmeof.Spec.Instances = 0
+ cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
+ r := newReconcile(cCtx, cl)
+ fakeRecorder := record.NewFakeRecorder(5)
+ r.recorder = fakeRecorder
+
+ res, err := r.Reconcile(ctx, req)
+ assert.Error(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assert.Contains(t, err.Error(), "invalid configuration")
+
+ assert.Len(t, fakeRecorder.Events, 1)
+ event := <-fakeRecorder.Events
+ assert.Contains(t, event, "ReconcileFailed")
+ assert.Contains(t, event, "invalid configuration")
+ })
+
+ t.Run("group empty should error", func(t *testing.T) {
+ cCtx := newContext(successExecutor(t))
+ nvmeof := baseCephNVMeOFGateway()
+ nvmeof.Spec.Group = ""
+ cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
+ r := newReconcile(cCtx, cl)
+ fakeRecorder := record.NewFakeRecorder(5)
+ r.recorder = fakeRecorder
+
+ res, err := r.Reconcile(ctx, req)
+ assert.Error(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assert.Contains(t, err.Error(), "invalid configuration")
+
+ assert.Len(t, fakeRecorder.Events, 1)
+ event := <-fakeRecorder.Events
+ assert.Contains(t, event, "ReconcileFailed")
+ assert.Contains(t, event, "invalid configuration")
+ })
+ })
+
+ assertCephNVMeOFGatewayReady := func(t *testing.T, r *ReconcileCephNVMeOFGateway, names ...string) {
+ t.Helper()
+
+ if len(names) == 0 {
+ names = []string{name}
+ }
+
+ for _, n := range names {
+ cephNVMeOFGateway := &cephv1.CephNVMeOFGateway{}
+ err := r.client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: n}, cephNVMeOFGateway)
+ assert.NoError(t, err)
+ assert.Equal(t, "Ready", cephNVMeOFGateway.Status.Phase, cephNVMeOFGateway)
+ }
+ }
+
+ assertResourcesExist := func(t *testing.T, cCtx *clusterd.Context, names ...string) {
+ t.Helper()
+
+ depNames := []string{}
+ deps, err := cCtx.Clientset.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{})
+ assert.NoError(t, err)
+ for _, dep := range deps.Items {
+ depNames = append(depNames, dep.Name)
+ }
+ assert.ElementsMatch(t, names, depNames)
+
+ svcNames := []string{}
+ svcs, err := cCtx.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{})
+ assert.NoError(t, err)
+ for _, dep := range svcs.Items {
+ svcNames = append(svcNames, dep.Name)
+ }
+ assert.ElementsMatch(t, names, svcNames)
+ }
+
+ t.Run("run one nvmeof gateway", func(t *testing.T) {
+ cCtx := newContext(successExecutor(t))
+ cl := newControllerClient(baseCephNVMeOFGateway(), cephClusterReady(cCtx))
+ r := newReconcile(cCtx, cl)
+
+ t.Run("initial reconcile", func(t *testing.T) {
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r)
+ assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a")
+ })
+
+ t.Run("double reconcile", func(t *testing.T) {
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r)
+ assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a")
+ })
+ })
+
+ t.Run("run multiple nvmeof gateways", func(t *testing.T) {
+ cCtx := newContext(successExecutor(t))
+ nvmeof := baseCephNVMeOFGateway()
+ nvmeof.Spec.Instances = 3
+ cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
+ r := newReconcile(cCtx, cl)
+
+ t.Run("initial reconcile", func(t *testing.T) {
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r)
+ assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
+ })
+
+ t.Run("double reconcile", func(t *testing.T) {
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r)
+ assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
+ })
+ })
+
+ t.Run("scale down nvmeof gateways", func(t *testing.T) {
+ t.Run("scale from 3 to 2 gateways", func(t *testing.T) {
+ cCtx := newContext(successExecutor(t))
+ nvmeof := baseCephNVMeOFGateway()
+ nvmeof.Spec.Instances = 3
+ cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
+ r := newReconcile(cCtx, cl)
+
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r)
+ assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
+
+ err = cl.Get(ctx, client.ObjectKeyFromObject(nvmeof), nvmeof)
+ assert.NoError(t, err)
+ nvmeof.Spec.Instances = 2
+ err = cl.Update(ctx, nvmeof, &client.UpdateOptions{})
+ assert.NoError(t, err)
+
+ res, err = r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r)
+ assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b")
+ })
+
+ t.Run("scale from 3 to 1 gateway", func(t *testing.T) {
+ cCtx := newContext(successExecutor(t))
+ nvmeof := baseCephNVMeOFGateway()
+ nvmeof.Spec.Instances = 3
+ cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
+ r := newReconcile(cCtx, cl)
+
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r)
+ assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
+
+ err = cl.Get(ctx, client.ObjectKeyFromObject(nvmeof), nvmeof)
+ assert.NoError(t, err)
+ nvmeof.Spec.Instances = 1
+ err = cl.Update(ctx, nvmeof, &client.UpdateOptions{})
+ assert.NoError(t, err)
+
+ res, err = r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r)
+ assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a")
+ })
+ })
+
+ t.Run("multiple CephNVMeOFGateway clusters", func(t *testing.T) {
+ nvmeof1 := baseCephNVMeOFGateway()
+ nvmeof1.Spec.Instances = 3
+ nvmeof2 := baseCephNVMeOFGateway()
+ nvmeof2.Spec.Instances = 2
+ nvmeof2.Name = "nvmeof2"
+
+ cCtx := newContext(successExecutor(t))
+ cl := newControllerClient(nvmeof1, nvmeof2, cephClusterReady(cCtx))
+ r := newReconcile(cCtx, cl)
+
+ req2 := reconcile.Request{
+ NamespacedName: types.NamespacedName{
+ Name: "nvmeof2",
+ Namespace: namespace,
+ },
+ }
+
+ t.Run("reconcile first CephNVMeOFGateway cluster", func(t *testing.T) {
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r, "my-nvmeof") // first cluster should be ready
+ assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
+ })
+
+ t.Run("reconcile second CephNVMeOFGateway cluster", func(t *testing.T) {
+ res, err := r.Reconcile(ctx, req2)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+ assertCephNVMeOFGatewayReady(t, r, "my-nvmeof", "nvmeof2")
+ assertResourcesExist(t, cCtx,
+ "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c",
+ "rook-ceph-nvmeof-nvmeof2-a", "rook-ceph-nvmeof-nvmeof2-b",
+ )
+ })
+
+ t.Run("scale down first CephNVMeOFGateway cluster (3 to 1)", func(t *testing.T) {
+ err := cl.Get(ctx, client.ObjectKeyFromObject(nvmeof1), nvmeof1)
+ assert.NoError(t, err)
+ nvmeof1.Spec.Instances = 1
+ err = cl.Update(ctx, nvmeof1, &client.UpdateOptions{})
+ assert.NoError(t, err)
+
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+
+ assertCephNVMeOFGatewayReady(t, r, "my-nvmeof", "nvmeof2")
+ assertResourcesExist(t, cCtx,
+ "rook-ceph-nvmeof-my-nvmeof-a",
+ "rook-ceph-nvmeof-nvmeof2-a", "rook-ceph-nvmeof-nvmeof2-b",
+ )
+ })
+ })
+}
+
+func TestNVMeOFKeyRotation(t *testing.T) {
+ ctx := context.TODO()
+ var (
+ name = "my-nvmeof"
+ namespace = "rook-ceph"
+ )
+ // Set DEBUG logging
+ t.Setenv("ROOK_LOG_LEVEL", "DEBUG")
+
+ // Mock version to avoid deployment updates (filesystem mirror pattern)
+ currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) {
+ return &version.CephVersion{Major: 20, Minor: 2, Extra: 0}, &version.CephVersion{Major: 20, Minor: 2, Extra: 0}, nil
+ }
+
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: namespace,
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ TypeMeta: controllerTypeMeta,
+ }
+
+ cephCluster := &cephv1.CephCluster{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: namespace,
+ Namespace: namespace,
+ },
+ Spec: cephv1.ClusterSpec{
+ Security: cephv1.ClusterSecuritySpec{
+ CephX: cephv1.ClusterCephxConfig{
+ Daemon: cephv1.CephxConfig{},
+ },
+ },
+ },
+ Status: cephv1.ClusterStatus{
+ Phase: k8sutil.ReadyStatus,
+ CephStatus: &cephv1.CephStatus{
+ Health: "HEALTH_OK",
+ },
+ },
+ }
+
+ object := []runtime.Object{
+ nvmeof,
+ cephCluster,
+ }
+
+ s := scheme.Scheme
+ s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephNVMeOFGateway{})
+ s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
+
+ executor := &exectest.MockExecutor{
+ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
+ if command == "ceph" {
+ if args[0] == "status" {
+ return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
+ }
+ if args[0] == "nvme-gw" && args[1] == "create" {
+ return "", nil
+ }
+ if args[0] == "nvme-gw" && args[1] == "show" {
+ return "", nil
+ }
+ }
+ return "", nil
+ },
+ }
+ clientset := test.New(t, 3)
+ c := &clusterd.Context{
+ Executor: executor,
+ RookClientset: rookclient.NewSimpleClientset(),
+ Clientset: clientset,
+ }
+
+ cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+ r := &ReconcileCephNVMeOFGateway{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)}
+
+ req := reconcile.Request{
+ NamespacedName: types.NamespacedName{
+ Name: name,
+ Namespace: namespace,
+ },
+ }
+
+ t.Run("first reconcile", func(t *testing.T) {
+ secrets := map[string][]byte{
+ "fsid": []byte(name),
+ "mon-secret": []byte("monsecret"),
+ "admin-secret": []byte("adminsecret"),
+ }
+ secret := &v1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "rook-ceph-mon",
+ Namespace: namespace,
+ },
+ Data: secrets,
+ Type: k8sutil.RookType,
+ }
+ _, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
+ assert.NoError(t, err)
+
+ _, err = r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+
+ nvmeofResult := cephv1.CephNVMeOFGateway{}
+ err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
+ assert.NoError(t, err)
+ assert.Equal(t, uint32(1), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
+ assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
+ })
+
+ t.Run("subsequent reconcile - retain cephx status", func(t *testing.T) {
+ r := &ReconcileCephNVMeOFGateway{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)}
+ _, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ nvmeofResult := cephv1.CephNVMeOFGateway{}
+ err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
+ assert.NoError(t, err)
+ assert.Equal(t, uint32(1), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
+ assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
+ })
+
+ t.Run("brownfield reconcile - retain unknown cephx status", func(t *testing.T) {
+ nvmeofResult := cephv1.CephNVMeOFGateway{}
+ err := cl.Get(ctx, req.NamespacedName, &nvmeofResult)
+ assert.NoError(t, err)
+ nvmeofResult.Status.Cephx.Daemon = cephv1.CephxStatus{}
+ err = cl.Update(ctx, &nvmeofResult)
+ assert.NoError(t, err)
+
+ _, err = r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+
+ err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
+ assert.NoError(t, err)
+ assert.Equal(t, cephv1.CephxStatus{}, nvmeofResult.Status.Cephx.Daemon)
+ })
+
+ t.Run("rotate key - brownfield unknown status becomes known", func(t *testing.T) {
+ cluster := cephv1.CephCluster{}
+ err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster)
+ assert.NoError(t, err)
+ cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{
+ KeyRotationPolicy: "KeyGeneration",
+ KeyGeneration: 2,
+ }
+ err = cl.Update(ctx, &cluster)
+ assert.NoError(t, err)
+
+ _, err = r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+
+ nvmeofResult := cephv1.CephNVMeOFGateway{}
+ err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
+ assert.NoError(t, err)
+ assert.Equal(t, uint32(2), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
+ assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
+ })
+
+ t.Run("brownfield reconcile - no further rotation happens", func(t *testing.T) {
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, res.RequeueAfter, time.Duration(0))
+
+ nvmeofResult := cephv1.CephNVMeOFGateway{}
+ err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
+ assert.NoError(t, err)
+ assert.Equal(t, uint32(2), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
+ assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
+ })
+
+ t.Run("rotate key - cephx status updated", func(t *testing.T) {
+ cluster := cephv1.CephCluster{}
+ err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster)
+ assert.NoError(t, err)
+ cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{
+ KeyRotationPolicy: "KeyGeneration",
+ KeyGeneration: 3,
+ }
+ err = cl.Update(ctx, &cluster)
+ assert.NoError(t, err)
+
+ _, err = r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+
+ nvmeofResult := cephv1.CephNVMeOFGateway{}
+ err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
+ assert.NoError(t, err)
+ assert.Equal(t, uint32(3), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
+ assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
+ })
+}
+
+func TestNVMeOFConfigGeneration(t *testing.T) {
+ t.Run("defaults", func(t *testing.T) {
+ config, err := getNVMeOFGatewayConfig("pool-a", "pod-a", "10.0.0.1", "ana-a", nil)
+ assert.NoError(t, err)
+
+ cfg, err := ini.Load([]byte(config))
+ assert.NoError(t, err)
+ assert.Equal(t, "pod-a", cfg.Section("gateway").Key("name").String())
+ assert.Equal(t, "ana-a", cfg.Section("gateway").Key("group").String())
+ assert.Equal(t, "10.0.0.1", cfg.Section("gateway").Key("addr").String())
+ assert.Equal(t, "5500", cfg.Section("gateway").Key("port").String())
+ assert.Equal(t, "0.0.0.0", cfg.Section("discovery").Key("addr").String())
+ assert.Equal(t, "8009", cfg.Section("discovery").Key("port").String())
+ assert.Equal(t, "admin", cfg.Section("ceph").Key("id").String())
+ assert.Equal(t, "pool-a", cfg.Section("ceph").Key("pool").String())
+ assert.Equal(t, "4096", cfg.Section("spdk").Key("mem_size").String())
+ assert.Equal(t, "5499", cfg.Section("monitor").Key("port").String())
+ })
+
+ t.Run("overrides and new section", func(t *testing.T) {
+ userConfig := map[string]map[string]string{
+ "gateway": {
+ "port": "6600",
+ "enable_auth": "True",
+ },
+ "ceph": {
+ "pool": "pool-b",
+ },
+ "custom": {
+ "foo": "bar",
+ },
+ }
+ config, err := getNVMeOFGatewayConfig("pool-a", "pod-a", "10.0.0.1", "ana-a", userConfig)
+ assert.NoError(t, err)
+
+ cfg, err := ini.Load([]byte(config))
+ assert.NoError(t, err)
+ assert.Equal(t, "6600", cfg.Section("gateway").Key("port").String())
+ assert.Equal(t, "True", cfg.Section("gateway").Key("enable_auth").String())
+ assert.Equal(t, "pool-b", cfg.Section("ceph").Key("pool").String())
+ assert.Equal(t, "bar", cfg.Section("custom").Key("foo").String())
+ })
+}
+
+func TestNVMeOFConfigMapGeneration(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Pool: "pool-a",
+ Group: "group-a",
+ Instances: 1,
+ NVMeOFConfig: map[string]map[string]string{
+ "gateway": {
+ "port": "5511",
+ },
+ },
+ },
+ TypeMeta: controllerTypeMeta,
+ }
+
+ r := &ReconcileCephNVMeOFGateway{}
+ configMap, err := r.generateConfigMap(nvmeof, "a")
+ assert.NoError(t, err)
+ assert.Equal(t, "rook-ceph-nvmeof-my-nvmeof-a-config", configMap.Name)
+ assert.Equal(t, "rook-ceph", configMap.Namespace)
+ assert.Contains(t, configMap.Data, "config")
+
+ cfg, err := ini.Load([]byte(configMap.Data["config"]))
+ assert.NoError(t, err)
+ assert.Equal(t, instanceName(nvmeof, "a"), cfg.Section("gateway").Key("name").String())
+ assert.Equal(t, "@@POD_IP@@", cfg.Section("gateway").Key("addr").String())
+ assert.Equal(t, "group-a", cfg.Section("gateway").Key("group").String())
+ assert.Equal(t, "5511", cfg.Section("gateway").Key("port").String())
+ assert.Equal(t, "pool-a", cfg.Section("ceph").Key("pool").String())
+}
diff --git a/pkg/operator/ceph/nvmeof/spec.go b/pkg/operator/ceph/nvmeof/spec.go
new file mode 100644
index 000000000..06b415927
--- /dev/null
+++ b/pkg/operator/ceph/nvmeof/spec.go
@@ -0,0 +1,474 @@
+/*
+Copyright 2025 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nvmeof
+
+import (
+ _ "embed"
+ "fmt"
+ "strings"
+
+ "github.com/pkg/errors"
+ cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
+ "github.com/rook/rook/pkg/operator/ceph/controller"
+ "github.com/rook/rook/pkg/operator/k8sutil"
+ apps "k8s.io/api/apps/v1"
+ v1 "k8s.io/api/core/v1"
+ kerrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/intstr"
+ "k8s.io/apimachinery/pkg/util/validation"
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+const (
+ AppName = "rook-ceph-nvmeof"
+ nvmeofIOPort = 4420
+ nvmeofGatewayPort = 5500
+ nvmeofMonitorPort = 5499
+ nvmeofDiscoveryPort = 8009
+ defaultNVMeOFImage = "quay.io/ceph/nvmeof:1.5"
+ configKey = "config"
+ serviceAccountName = "rook-ceph-nvmeof"
+)
+
+// getPorts returns the configured ports with defaults
+func getPorts(nvmeof *cephv1.CephNVMeOFGateway) (ioPort, gatewayPort, monitorPort, discoveryPort int32) {
+ ioPort = nvmeofIOPort
+ gatewayPort = nvmeofGatewayPort
+ monitorPort = nvmeofMonitorPort
+ discoveryPort = nvmeofDiscoveryPort
+
+ if nvmeof.Spec.Ports != nil {
+ if nvmeof.Spec.Ports.IOPort != 0 {
+ ioPort = nvmeof.Spec.Ports.IOPort
+ }
+ if nvmeof.Spec.Ports.GatewayPort != 0 {
+ gatewayPort = nvmeof.Spec.Ports.GatewayPort
+ }
+ if nvmeof.Spec.Ports.MonitorPort != 0 {
+ monitorPort = nvmeof.Spec.Ports.MonitorPort
+ }
+ if nvmeof.Spec.Ports.DiscoveryPort != 0 {
+ discoveryPort = nvmeof.Spec.Ports.DiscoveryPort
+ }
+ }
+
+ return ioPort, gatewayPort, monitorPort, discoveryPort
+}
+
+//go:embed connectionconfig.sh
+var connectionConfigScript string
+
+func (r *ReconcileCephNVMeOFGateway) generateCephNVMeOFService(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) *v1.Service {
+ labels := getLabels(nvmeof, daemonID)
+ serviceName := instanceName(nvmeof, daemonID)
+ ioPort, gatewayPort, monitorPort, discoveryPort := getPorts(nvmeof)
+
+ svc := &v1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: serviceName,
+ Namespace: nvmeof.Namespace,
+ Labels: labels,
+ },
+ Spec: v1.ServiceSpec{
+ Selector: labels,
+ Ports: []v1.ServicePort{
+ {
+ Name: "io",
+ Port: ioPort,
+ TargetPort: intstr.FromInt(int(ioPort)),
+ Protocol: v1.ProtocolTCP,
+ },
+ {
+ Name: "gateway",
+ Port: gatewayPort,
+ TargetPort: intstr.FromInt(int(gatewayPort)),
+ Protocol: v1.ProtocolTCP,
+ },
+ {
+ Name: "monitor",
+ Port: monitorPort,
+ TargetPort: intstr.FromInt(int(monitorPort)),
+ Protocol: v1.ProtocolTCP,
+ },
+ {
+ Name: "discovery",
+ Port: discoveryPort,
+ TargetPort: intstr.FromInt(int(discoveryPort)),
+ Protocol: v1.ProtocolTCP,
+ },
+ },
+ },
+ }
+
+ hostNetwork := nvmeof.IsHostNetwork(r.cephClusterSpec)
+ if hostNetwork {
+ svc.Spec.ClusterIP = v1.ClusterIPNone
+ }
+
+ return svc
+}
+
+func (r *ReconcileCephNVMeOFGateway) createCephNVMeOFService(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) error {
+ s := r.generateCephNVMeOFService(nvmeof, daemonID)
+
+ err := controllerutil.SetControllerReference(nvmeof, s, r.scheme)
+ if err != nil {
+ logger.Errorf("failed to set owner reference: %v", err)
+ return errors.Wrapf(err, "failed to set owner reference to ceph nvmeof gateway %q", s)
+ }
+
+ svc, err := r.context.Clientset.CoreV1().Services(nvmeof.Namespace).Create(r.opManagerContext, s, metav1.CreateOptions{})
+ if err != nil {
+ if !kerrors.IsAlreadyExists(err) {
+ logger.Errorf("failed to create service: %v", err)
+ return errors.Wrap(err, "failed to create nvmeof gateway service")
+ }
+ logger.Infof("ceph nvmeof gateway service already created")
+ return nil
+ }
+
+ _, _, _, discoveryPort := getPorts(nvmeof)
+ logger.Infof("ceph nvmeof gateway service running at %s:%d", svc.Spec.ClusterIP, discoveryPort)
+ return nil
+}
+
+func (r *ReconcileCephNVMeOFGateway) makeDeployment(nvmeof *cephv1.CephNVMeOFGateway, daemonID, configMapName, configHash string) (*apps.Deployment, error) {
+ if r == nil {
+ logger.Errorf("receiver is nil")
+ return nil, errors.New("receiver is nil")
+ }
+ if r.clusterInfo == nil {
+ logger.Errorf("clusterInfo is nil")
+ return nil, errors.New("clusterInfo is nil")
+ }
+ if r.clusterInfo.CephVersion.String() == "" {
+ logger.Errorf("CephVersion is empty")
+ return nil, errors.New("CephVersion is empty")
+ }
+
+ resourceName := instanceName(nvmeof, daemonID)
+ deployment := &apps.Deployment{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: resourceName,
+ Namespace: nvmeof.Namespace,
+ Labels: getLabels(nvmeof, daemonID),
+ },
+ }
+
+ hostNetwork := nvmeof.IsHostNetwork(r.cephClusterSpec)
+
+ k8sutil.AddRookVersionLabelToDeployment(deployment)
+ controller.AddCephVersionLabelToDeployment(r.clusterInfo.CephVersion, deployment)
+ nvmeof.Spec.Annotations.ApplyToObjectMeta(&deployment.ObjectMeta)
+ nvmeof.Spec.Labels.ApplyToObjectMeta(&deployment.ObjectMeta)
+
+ cephConfigVol, cephConfigMount := controller.ConfGeneratedInPodVolumeAndMount()
+ gatewayConfigVol, gatewayConfigMount := gatewayConfigVolumeAndMount(configMapName)
+
+ initContainer := r.createCephConfigInitContainer(nvmeof, daemonID, gatewayConfigMount)
+ daemonContainer := r.daemonContainer(nvmeof, cephConfigMount)
+
+ gatewayName := instanceName(nvmeof, daemonID)
+ podSpec := v1.PodSpec{
+ InitContainers: []v1.Container{
+ initContainer,
+ },
+ Containers: []v1.Container{
+ daemonContainer,
+ },
+ RestartPolicy: v1.RestartPolicyAlways,
+ Volumes: []v1.Volume{
+ cephConfigVol,
+ adminKeyringVolume(),
+ gatewayConfigVol,
+ },
+ HostNetwork: hostNetwork,
+ PriorityClassName: nvmeof.Spec.PriorityClassName,
+ SecurityContext: &v1.PodSecurityContext{},
+ ServiceAccountName: serviceAccountName,
+ }
+
+ // Ensure a stable hostname inside the pod (does NOT make the pod IP stable).
+ // This matches the gateway instance name used elsewhere by the operator.
+ if errs := validation.IsDNS1123Label(gatewayName); len(errs) == 0 {
+ podSpec.Hostname = gatewayName
+ } else {
+ logger.Warningf("not setting pod hostname %q (must be a DNS-1123 label): %v", gatewayName, errs)
+ }
+
+ k8sutil.AddUnreachableNodeToleration(&podSpec)
+
+ if hostNetwork {
+ podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
+ }
+ nvmeof.Spec.Placement.ApplyToPodSpec(&podSpec)
+
+ podTemplateSpec := v1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: resourceName,
+ Labels: getLabels(nvmeof, daemonID),
+ Annotations: map[string]string{
+ "config-hash": configHash,
+ },
+ },
+ Spec: podSpec,
+ }
+
+ if hostNetwork {
+ podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
+ } else if r.cephClusterSpec.Network.IsMultus() {
+ if err := k8sutil.ApplyMultus(r.clusterInfo.Namespace, &r.cephClusterSpec.Network, &podTemplateSpec.ObjectMeta); err != nil {
+ logger.Errorf("failed to apply Multus configuration: %v", err)
+ return nil, err
+ }
+ }
+
+ nvmeof.Spec.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
+ nvmeof.Spec.Labels.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
+
+ replicas := int32(1)
+ revisionHistoryLimit := controller.RevisionHistoryLimit()
+ deployment.Spec = apps.DeploymentSpec{
+ RevisionHistoryLimit: revisionHistoryLimit,
+ Selector: &metav1.LabelSelector{MatchLabels: getLabels(nvmeof, daemonID)},
+ Template: podTemplateSpec,
+ Replicas: &replicas,
+ }
+
+ return deployment, nil
+}
+
+// createCephConfigInitContainer creates a minimal init container that generates
+// /etc/ceph/ceph.conf, copies the admin keyring, and copies nvmeof.conf from the ConfigMap.
+// This is needed for the gateway to connect to the Ceph cluster.
+func (r *ReconcileCephNVMeOFGateway) createCephConfigInitContainer(nvmeof *cephv1.CephNVMeOFGateway, daemonID string, gatewayConfigMount v1.VolumeMount) v1.Container {
+ _, cephConfigMount := controller.ConfGeneratedInPodVolumeAndMount()
+
+ cephImage := r.cephClusterSpec.CephVersion.Image
+ imagePullPolicy := controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy)
+
+ // Build Ceph CLI arguments using Rook's helper functions
+ // Use AdminFlags since the init container uses admin keyring for Ceph commands
+ // Override the keyring path to use /etc/ceph/keyring (where the script copies it)
+ cephArgs := controller.AdminFlags(r.clusterInfo)
+ // Replace the keyring path with the one used by the init container script
+ for i, arg := range cephArgs {
+ if strings.HasPrefix(arg, "--keyring=") {
+ cephArgs[i] = "--keyring=/etc/ceph/keyring"
+ break
+ }
+ }
+
+ // Use the embedded script and pass Ceph CLI args via "$@"
+ script := connectionConfigScript
+
+ gatewayName := instanceName(nvmeof, daemonID)
+ poolName := nvmeof.Spec.Pool
+ anaGroup := nvmeof.Spec.Group
+
+ // Build environment variables using Rook's helper functions
+ // DaemonEnvVars already includes StoredMonHostEnvVars() which provides ROOK_CEPH_MON_HOST
+ envVars := controller.DaemonEnvVars(r.cephClusterSpec)
+ envVars = append(envVars,
+ v1.EnvVar{
+ Name: "GATEWAY_NAME",
+ Value: gatewayName,
+ },
+ v1.EnvVar{
+ Name: "POOL_NAME",
+ Value: poolName,
+ },
+ v1.EnvVar{
+ Name: "ANA_GROUP",
+ Value: anaGroup,
+ },
+ k8sutil.PodIPEnvVar("POD_IP"),
+ )
+
+ privileged := true
+ container := v1.Container{
+ Name: "generate-ceph-conf",
+ Image: cephImage,
+ ImagePullPolicy: imagePullPolicy,
+ Command: []string{
+ "/bin/bash",
+ "-c",
+ script,
+ },
+ Args: cephArgs,
+ Env: envVars,
+ VolumeMounts: []v1.VolumeMount{
+ adminKeyringVolumeMount(),
+ cephConfigMount,
+ gatewayConfigMount,
+ },
+ Resources: nvmeof.Spec.Resources,
+ SecurityContext: &v1.SecurityContext{
+ Privileged: &privileged,
+ Capabilities: &v1.Capabilities{
+ Add: []v1.Capability{
+ "SYS_ADMIN",
+ },
+ Drop: []v1.Capability{
+ "NET_RAW",
+ },
+ },
+ },
+ }
+ return container
+}
+
+func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGateway, cephConfigMount v1.VolumeMount) v1.Container {
+ privileged := true
+ container := v1.Container{
+ Name: "nvmeof-gateway",
+ Args: []string{
+ "-c",
+ "/etc/ceph/nvmeof.conf",
+ },
+
+ Image: getNVMeOFImage(nvmeof),
+ ImagePullPolicy: v1.PullIfNotPresent,
+ VolumeMounts: []v1.VolumeMount{
+ cephConfigMount,
+ },
+ Env: func() []v1.EnvVar {
+ envVars := controller.DaemonEnvVars(r.cephClusterSpec)
+ // Build CEPH_ARGS using DefaultFlags which includes fsid, keyring, logging flags, and mon host flags
+ cephArgs := cephconfig.DefaultFlags(r.clusterInfo.FSID, "/etc/ceph/keyring")
+ cephArgsStr := strings.Join(cephArgs, " ")
+ logger.Infof("CEPH_ARGS for nvmeof gateway: %s", cephArgsStr)
+ envVars = append(envVars, v1.EnvVar{
+ Name: "CEPH_ARGS",
+ Value: cephArgsStr,
+ })
+ return envVars
+ }(),
+
+ Ports: func() []v1.ContainerPort {
+ ioPort, gatewayPort, monitorPort, discoveryPort := getPorts(nvmeof)
+ return []v1.ContainerPort{
+ {
+ Name: "io",
+ ContainerPort: ioPort,
+ Protocol: v1.ProtocolTCP,
+ },
+ {
+ Name: "gateway",
+ ContainerPort: gatewayPort,
+ Protocol: v1.ProtocolTCP,
+ },
+ {
+ Name: "monitor",
+ ContainerPort: monitorPort,
+ Protocol: v1.ProtocolTCP,
+ },
+ {
+ Name: "discovery",
+ ContainerPort: discoveryPort,
+ Protocol: v1.ProtocolTCP,
+ },
+ }
+ }(),
+
+ Resources: nvmeof.Spec.Resources,
+ SecurityContext: &v1.SecurityContext{
+ Privileged: &privileged,
+ Capabilities: &v1.Capabilities{
+ Add: []v1.Capability{
+ "SYS_ADMIN",
+ },
+ Drop: []v1.Capability{
+ "NET_RAW",
+ },
+ },
+ },
+ }
+
+ if nvmeof.Spec.LivenessProbe != nil && !nvmeof.Spec.LivenessProbe.Disabled && nvmeof.Spec.LivenessProbe.Probe == nil {
+ container.LivenessProbe = r.defaultLivenessProbe(nvmeof)
+ }
+ result := cephconfig.ConfigureLivenessProbe(container, nvmeof.Spec.LivenessProbe)
+ return result
+}
+
+func (r *ReconcileCephNVMeOFGateway) defaultLivenessProbe(nvmeof *cephv1.CephNVMeOFGateway) *v1.Probe {
+ ioPort, _, _, _ := getPorts(nvmeof)
+ return controller.GenerateLivenessProbeTcpPort(ioPort, 10)
+}
+
+func getLabels(n *cephv1.CephNVMeOFGateway, daemonID string) map[string]string {
+ return controller.CephDaemonAppLabels(
+ AppName, n.Namespace, "nvmeof", n.Name+"-"+daemonID, n.Name,
+ "cephnvmeofgateways.ceph.rook.io", true,
+ )
+}
+
+func instanceName(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) string {
+ return fmt.Sprintf("%s-%s-%s", AppName, nvmeof.Name, daemonID)
+}
+
+// getNVMeOFImage returns the image to use for the NVMe-oF gateway.
+// If the image is specified in the spec, it is used; otherwise, the default image is returned.
+func getNVMeOFImage(nvmeof *cephv1.CephNVMeOFGateway) string {
+ if nvmeof.Spec.Image != "" {
+ return nvmeof.Spec.Image
+ }
+ return defaultNVMeOFImage
+}
+
+func gatewayConfigVolumeAndMount(configConfigMap string) (v1.Volume, v1.VolumeMount) {
+ // Mount to /config temporarily - init container will copy to /etc/ceph/nvmeof.conf
+ cfgDir := "/config"
+ cfgVolName := "gateway-config"
+ configMapSource := &v1.ConfigMapVolumeSource{
+ LocalObjectReference: v1.LocalObjectReference{Name: configConfigMap},
+ Items: []v1.KeyToPath{{Key: configKey, Path: "nvmeof.conf"}},
+ }
+ v := v1.Volume{Name: cfgVolName, VolumeSource: v1.VolumeSource{ConfigMap: configMapSource}}
+ m := v1.VolumeMount{Name: cfgVolName, MountPath: cfgDir, ReadOnly: true}
+ return v, m
+}
+
+func adminKeyringVolume() v1.Volume {
+ mode := int32(292)
+ return v1.Volume{
+ Name: "ceph-admin-keyring",
+ VolumeSource: v1.VolumeSource{
+ Secret: &v1.SecretVolumeSource{
+ SecretName: "rook-ceph-admin-keyring",
+ Items: []v1.KeyToPath{
+ {
+ Key: "keyring",
+ Path: "keyring",
+ Mode: &mode,
+ },
+ },
+ },
+ },
+ }
+}
+
+func adminKeyringVolumeMount() v1.VolumeMount {
+ return v1.VolumeMount{
+ Name: "ceph-admin-keyring",
+ MountPath: "/tmp/ceph/",
+ ReadOnly: true,
+ }
+}
diff --git a/pkg/operator/ceph/nvmeof/spec_test.go b/pkg/operator/ceph/nvmeof/spec_test.go
new file mode 100644
index 000000000..3cef9141e
--- /dev/null
+++ b/pkg/operator/ceph/nvmeof/spec_test.go
@@ -0,0 +1,536 @@
+/*
+Copyright 2025 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nvmeof
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
+ "github.com/rook/rook/pkg/client/clientset/versioned/scheme"
+ "github.com/rook/rook/pkg/clusterd"
+ cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+ cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
+ cephTest "github.com/rook/rook/pkg/operator/ceph/test"
+ cephver "github.com/rook/rook/pkg/operator/ceph/version"
+ optest "github.com/rook/rook/pkg/operator/test"
+ exectest "github.com/rook/rook/pkg/util/exec/test"
+ "github.com/stretchr/testify/assert"
+ v1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func newDeploymentSpecTest(t *testing.T) (*ReconcileCephNVMeOFGateway, string) {
+ clientset := optest.New(t, 1)
+ c := &clusterd.Context{
+ Executor: &exectest.MockExecutor{},
+ RookClientset: rookclient.NewSimpleClientset(),
+ Clientset: clientset,
+ }
+
+ s := scheme.Scheme
+ object := []runtime.Object{
+ &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: namespace,
+ },
+ TypeMeta: controllerTypeMeta,
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ },
+ }
+ cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
+
+ r := &ReconcileCephNVMeOFGateway{
+ client: cl,
+ scheme: scheme.Scheme,
+ context: c,
+ clusterInfo: &cephclient.ClusterInfo{
+ FSID: "myfsid",
+ CephVersion: cephver.Squid,
+ },
+ cephClusterSpec: &cephv1.ClusterSpec{
+ CephVersion: cephv1.CephVersionSpec{
+ Image: "quay.io/ceph/ceph:v19",
+ },
+ },
+ }
+
+ configHash := "dcb0d2f5f5e86ec4929d8243cd640b8154165f8ff9b89809964fc7993e9b0101"
+
+ return r, configHash
+}
+
+func TestDeploymentSpec(t *testing.T) {
+ t.Run("basic", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 3,
+ Resources: v1.ResourceRequirements{
+ Limits: v1.ResourceList{
+ v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
+ v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
+ },
+ Requests: v1.ResourceList{
+ v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
+ v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
+ },
+ },
+ PriorityClassName: "my-priority-class",
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, d.Spec.Template.Annotations)
+ assert.Equal(t, configHash, d.Spec.Template.Annotations["config-hash"])
+
+ // Verify deployment labels contain Ceph requirements
+ daemonID := nvmeof.Name + "-0"
+ cephTest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels,
+ "nvmeof", daemonID, AppName, nvmeof.Namespace, nvmeof.Name,
+ "cephnvmeofgateways.ceph.rook.io", "ceph-nvmeof")
+
+ podTemplate := optest.NewPodTemplateSpecTester(t, &d.Spec.Template)
+ podTemplate.RunFullSuite(
+ AppName,
+ optest.ResourceLimitExpectations{
+ CPUResourceLimit: "500",
+ MemoryResourceLimit: "1Ki",
+ CPUResourceRequest: "200",
+ MemoryResourceRequest: "512",
+ },
+ )
+ assert.Equal(t, "my-priority-class", d.Spec.Template.Spec.PriorityClassName)
+ assert.Equal(t, serviceAccountName, d.Spec.Template.Spec.ServiceAccountName)
+ })
+
+ t.Run("basic with default liveness-probe", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 3,
+ Resources: v1.ResourceRequirements{
+ Limits: v1.ResourceList{
+ v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
+ v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
+ },
+ Requests: v1.ResourceList{
+ v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
+ v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
+ },
+ },
+ PriorityClassName: "my-priority-class",
+ LivenessProbe: &cephv1.ProbeSpec{
+ Disabled: false,
+ },
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, d.Spec.Template.Annotations)
+
+ var nvmeofCont *v1.Container = nil
+ for i := range d.Spec.Template.Spec.Containers {
+ if d.Spec.Template.Spec.Containers[i].Name == "nvmeof-gateway" {
+ nvmeofCont = &d.Spec.Template.Spec.Containers[i]
+ break
+ }
+ }
+ assert.NotNil(t, nvmeofCont)
+ assert.NotNil(t, nvmeofCont.LivenessProbe)
+ assert.Equal(t, nvmeofCont.LivenessProbe.InitialDelaySeconds, int32(10))
+ assert.Equal(t, nvmeofCont.LivenessProbe.FailureThreshold, int32(10))
+ assert.GreaterOrEqual(t, nvmeofCont.LivenessProbe.TimeoutSeconds, int32(5))
+ })
+
+ t.Run("with host network", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+ r.cephClusterSpec.Network = cephv1.NetworkSpec{
+ HostNetwork: true,
+ }
+
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+ assert.True(t, d.Spec.Template.Spec.HostNetwork)
+ assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy)
+
+ svc := r.generateCephNVMeOFService(nvmeof, "0")
+ assert.Equal(t, v1.ClusterIPNone, svc.Spec.ClusterIP)
+ })
+
+ t.Run("with custom configmap ref", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ ConfigMapRef: "custom-configmap",
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+
+ configMapName := "custom-configmap"
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+ assert.NotEmpty(t, d.Spec.Template.Annotations)
+
+ var gatewayConfigVol *v1.Volume = nil
+ for i := range d.Spec.Template.Spec.Volumes {
+ if d.Spec.Template.Spec.Volumes[i].Name == "gateway-config" {
+ gatewayConfigVol = &d.Spec.Template.Spec.Volumes[i]
+ break
+ }
+ }
+ assert.NotNil(t, gatewayConfigVol)
+ assert.NotNil(t, gatewayConfigVol.ConfigMap)
+ assert.Equal(t, "custom-configmap", gatewayConfigVol.ConfigMap.Name)
+ })
+
+ t.Run("init container configuration", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+
+ assert.Len(t, d.Spec.Template.Spec.InitContainers, 1)
+ initCont := d.Spec.Template.Spec.InitContainers[0]
+ assert.Equal(t, "generate-ceph-conf", initCont.Name)
+ assert.Equal(t, r.cephClusterSpec.CephVersion.Image, initCont.Image)
+ assert.Equal(t, []string{"/bin/bash", "-c", connectionConfigScript}, initCont.Command)
+ assert.Contains(t, initCont.Args, "--keyring=/etc/ceph/keyring")
+
+ assertEnvVar(t, initCont.Env, "GATEWAY_NAME", instanceName(nvmeof, "0"))
+ assertEnvVar(t, initCont.Env, "POOL_NAME", nvmeof.Spec.Pool)
+ assertEnvVar(t, initCont.Env, "ANA_GROUP", nvmeof.Spec.Group)
+ assertEnvVarPresent(t, initCont.Env, "POD_IP")
+
+ volumeMountNames := []string{}
+ for _, vm := range initCont.VolumeMounts {
+ volumeMountNames = append(volumeMountNames, vm.Name)
+ }
+ assert.Contains(t, volumeMountNames, "ceph-admin-keyring")
+ assert.Contains(t, volumeMountNames, "gateway-config")
+ assert.Contains(t, volumeMountNames, "ceph-conf-emptydir")
+ })
+
+ t.Run("daemon container configuration", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+
+ assert.Len(t, d.Spec.Template.Spec.Containers, 1)
+ daemonCont := d.Spec.Template.Spec.Containers[0]
+ assert.Equal(t, "nvmeof-gateway", daemonCont.Name)
+ assert.Equal(t, nvmeof.Spec.Image, daemonCont.Image)
+
+ expectedCephArgs := strings.Join(cephconfig.DefaultFlags(r.clusterInfo.FSID, "/etc/ceph/keyring"), " ")
+ assertEnvVar(t, daemonCont.Env, "CEPH_ARGS", expectedCephArgs)
+
+ portNames := []string{}
+ for _, p := range daemonCont.Ports {
+ portNames = append(portNames, p.Name)
+ }
+ assert.Contains(t, portNames, "io")
+ assert.Contains(t, portNames, "gateway")
+ assert.Contains(t, portNames, "monitor")
+ assert.Contains(t, portNames, "discovery")
+
+ assert.NotNil(t, daemonCont.SecurityContext)
+ // SecurityContext should be set with privileged=true for NVMeOF gateway
+ assert.NotNil(t, daemonCont.SecurityContext.Privileged)
+ assert.True(t, *daemonCont.SecurityContext.Privileged)
+ })
+
+ t.Run("daemon container uses default image when not specified", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ // Image not specified - should use default
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+
+ assert.Len(t, d.Spec.Template.Spec.Containers, 1)
+ daemonCont := d.Spec.Template.Spec.Containers[0]
+ assert.Equal(t, "nvmeof-gateway", daemonCont.Name)
+ assert.Equal(t, "quay.io/ceph/nvmeof:1.5", daemonCont.Image)
+ })
+
+ t.Run("hostname set when instance name is valid", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+ assert.Equal(t, instanceName(nvmeof, "0"), d.Spec.Template.Spec.Hostname)
+ })
+
+ t.Run("hostname not set when instance name is invalid", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my_nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+ assert.Empty(t, d.Spec.Template.Spec.Hostname)
+ })
+
+ t.Run("service generation", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ },
+ }
+
+ r, _ := newDeploymentSpecTest(t)
+
+ svc := r.generateCephNVMeOFService(nvmeof, "0")
+ assert.Equal(t, "rook-ceph-nvmeof-my-nvmeof-0", svc.Name)
+ assert.Equal(t, "rook-ceph-test-ns", svc.Namespace)
+
+ // Verify service labels contain Ceph requirements
+ daemonID := nvmeof.Name + "-0"
+ cephTest.AssertLabelsContainCephRequirements(t, svc.Labels,
+ "nvmeof", daemonID, AppName, nvmeof.Namespace, nvmeof.Name,
+ "cephnvmeofgateways.ceph.rook.io", "ceph-nvmeof")
+
+ portNames := []string{}
+ for _, p := range svc.Spec.Ports {
+ portNames = append(portNames, p.Name)
+ }
+ assert.Contains(t, portNames, "io")
+ assert.Contains(t, portNames, "gateway")
+ assert.Contains(t, portNames, "monitor")
+ assert.Contains(t, portNames, "discovery")
+
+ for _, p := range svc.Spec.Ports {
+ switch p.Name {
+ case "io":
+ assert.Equal(t, int32(nvmeofIOPort), p.Port)
+ case "gateway":
+ assert.Equal(t, int32(nvmeofGatewayPort), p.Port)
+ case "monitor":
+ assert.Equal(t, int32(nvmeofMonitorPort), p.Port)
+ case "discovery":
+ assert.Equal(t, int32(nvmeofDiscoveryPort), p.Port)
+ }
+ }
+ })
+
+ t.Run("custom ports", func(t *testing.T) {
+ nvmeof := &cephv1.CephNVMeOFGateway{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-nvmeof",
+ Namespace: "rook-ceph-test-ns",
+ },
+ Spec: cephv1.NVMeOFGatewaySpec{
+ Image: "quay.io/ceph/nvmeof:1.5",
+ Pool: "nvmeof",
+ Group: "group-a",
+ Instances: 1,
+ Ports: &cephv1.NVMeOFGatewayPorts{
+ IOPort: 14420,
+ GatewayPort: 15500,
+ MonitorPort: 15499,
+ DiscoveryPort: 18009,
+ },
+ },
+ }
+
+ r, configHash := newDeploymentSpecTest(t)
+
+ configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
+ d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
+ assert.NoError(t, err)
+
+ daemonCont := d.Spec.Template.Spec.Containers[0]
+ assertContainerPort(t, daemonCont.Ports, "io", 14420)
+ assertContainerPort(t, daemonCont.Ports, "gateway", 15500)
+ assertContainerPort(t, daemonCont.Ports, "monitor", 15499)
+ assertContainerPort(t, daemonCont.Ports, "discovery", 18009)
+
+ svc := r.generateCephNVMeOFService(nvmeof, "0")
+ assertServicePort(t, svc.Spec.Ports, "io", 14420)
+ assertServicePort(t, svc.Spec.Ports, "gateway", 15500)
+ assertServicePort(t, svc.Spec.Ports, "monitor", 15499)
+ assertServicePort(t, svc.Spec.Ports, "discovery", 18009)
+ })
+}
+
+func assertEnvVar(t *testing.T, envVars []v1.EnvVar, name, value string) {
+ t.Helper()
+ for _, env := range envVars {
+ if env.Name == name {
+ assert.Equal(t, value, env.Value)
+ return
+ }
+ }
+ assert.Failf(t, "missing env var", "expected %s", name)
+}
+
+func assertEnvVarPresent(t *testing.T, envVars []v1.EnvVar, name string) {
+ t.Helper()
+ for _, env := range envVars {
+ if env.Name == name {
+ assert.NotNil(t, env.ValueFrom)
+ return
+ }
+ }
+ assert.Failf(t, "missing env var", "expected %s", name)
+}
+
+func assertServicePort(t *testing.T, ports []v1.ServicePort, name string, value int32) {
+ t.Helper()
+ for _, port := range ports {
+ if port.Name == name {
+ assert.Equal(t, value, port.Port)
+ return
+ }
+ }
+ assert.Failf(t, "missing port", "expected %s", name)
+}
+
+func assertContainerPort(t *testing.T, ports []v1.ContainerPort, name string, value int32) {
+ t.Helper()
+ for _, port := range ports {
+ if port.Name == name {
+ assert.Equal(t, value, port.ContainerPort)
+ return
+ }
+ }
+ assert.Failf(t, "missing port", "expected %s", name)
+}